diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt index 1092563ce04..41de40c46b3 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/AbstractPerformanceProjectsTest.kt @@ -5,25 +5,17 @@ package org.jetbrains.kotlin.idea.perf -import com.intellij.codeInsight.daemon.DaemonCodeAnalyzer -import com.intellij.codeInsight.daemon.DaemonCodeAnalyzerSettings -import com.intellij.codeInsight.daemon.impl.DaemonCodeAnalyzerImpl import com.intellij.codeInsight.daemon.impl.HighlightInfo import com.intellij.codeInsight.daemon.impl.IdentifierHighlighterPassFactory import com.intellij.codeInsight.lookup.LookupElement import com.intellij.codeInspection.InspectionProfileEntry -import com.intellij.ide.startup.impl.StartupManagerImpl import com.intellij.openapi.actionSystem.IdeActions -import com.intellij.openapi.application.runWriteAction import com.intellij.openapi.editor.EditorFactory import com.intellij.openapi.fileEditor.FileDocumentManager import com.intellij.openapi.fileEditor.FileEditorManager import com.intellij.openapi.project.DumbService import com.intellij.openapi.project.Project -import com.intellij.openapi.projectRoots.JavaSdk import com.intellij.openapi.projectRoots.Sdk -import com.intellij.openapi.projectRoots.impl.JavaAwareProjectJdkTableImpl -import com.intellij.openapi.startup.StartupManager import com.intellij.profile.codeInspection.ProjectInspectionProfileManager import com.intellij.psi.PsiDocumentManager import com.intellij.psi.PsiFile @@ -35,18 +27,20 @@ import com.intellij.util.ThrowableRunnable import com.intellij.util.indexing.UnindexedFilesUpdater import com.intellij.util.ui.UIUtil import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager -import org.jetbrains.kotlin.idea.framework.KotlinSdkType import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP import org.jetbrains.kotlin.idea.perf.Stats.Companion.runAndMeasure +import org.jetbrains.kotlin.idea.perf.util.PerformanceSuite.ApplicationScope.Companion.initApp +import org.jetbrains.kotlin.idea.perf.util.PerformanceSuite.ApplicationScope.Companion.initSdk +import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.initDefaultProfile +import org.jetbrains.kotlin.idea.perf.util.logMessage import org.jetbrains.kotlin.idea.test.invalidateLibraryCache import org.jetbrains.kotlin.idea.testFramework.* +import org.jetbrains.kotlin.idea.testFramework.TestApplicationManager import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.cleanupCaches import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.close import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.isAKotlinScriptFile import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.openFileInEditor import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.openFixture -import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction.SIMPLE_JAVA_MODULE -import org.jetbrains.kotlin.idea.util.getProjectJdkTableSafe import java.io.File abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { @@ -63,29 +57,11 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { override fun setUp() { super.setUp() - myApplication = TestApplicationManager.getInstance() - runWriteAction { - val jdkTableImpl = JavaAwareProjectJdkTableImpl.getInstanceEx() - val homePath = if (jdkTableImpl.internalJdk.homeDirectory!!.name == "jre") { - jdkTableImpl.internalJdk.homeDirectory!!.parent.path - } else { - jdkTableImpl.internalJdk.homePath!! - } - - val javaSdk = JavaSdk.getInstance() - jdk18 = javaSdk.createJdk("1.8", homePath) - val internal = javaSdk.createJdk("IDEA jdk", homePath) - - val jdkTable = getProjectJdkTableSafe() - jdkTable.addJdk(jdk18, testRootDisposable) - jdkTable.addJdk(internal, testRootDisposable) - KotlinSdkType.setUpIfNeeded() - } - - GradleProcessOutputInterceptor.install(testRootDisposable) + myApplication = initApp(testRootDisposable) + initSdk(testRootDisposable) } - protected fun warmUpProject(stats: Stats, vararg filesToHighlight: String, openProject: () -> Project) { + internal fun warmUpProject(stats: Stats, vararg filesToHighlight: String, openProject: () -> Project) { assertTrue(filesToHighlight.isNotEmpty()) val project = openProject() @@ -96,7 +72,6 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { } } finally { closeProject(project) - myApplication.setDataProvider(null) } } @@ -105,15 +80,9 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { RunAll( ThrowableRunnable { super.tearDown() }, ThrowableRunnable { - if (myProject != null) { - logMessage { "myProject is about to be closed" } - DaemonCodeAnalyzerSettings.getInstance().isImportHintEnabled = true // return default value to avoid unnecessary save - (StartupManager.getInstance(project()) as StartupManagerImpl).checkCleared() - (DaemonCodeAnalyzer.getInstance(project()) as DaemonCodeAnalyzerImpl).cleanupAfterTest() - closeProject(project()) - myApplication.setDataProvider(null) + myProject?.let { project -> + closeProject(project) myProject = null - logMessage { "myProject is closed" } } }).run() } @@ -123,14 +92,72 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { return if (lastIndexOf >= 0) fileName.substring(lastIndexOf + 1) else fileName } - protected fun perfOpenHelloWorld(stats: Stats, note: String = ""): Project = - perfOpenProject( - name = "helloKotlin", - stats = stats, - note = note, - path = "idea/testData/perfTest/helloKotlin", - openAction = SIMPLE_JAVA_MODULE - ) + private fun closeProject(project: Project) = myApplication.closeProject(project) + + protected fun perfOpenProject( + stats: Stats, + note: String = "", + fast: Boolean = false, + initializer: ProjectBuilder.() -> Unit, + ): Project { + val projectBuilder = ProjectBuilder().apply(initializer) + val name = projectBuilder.name + val openProjectOperation = projectBuilder.openProjectOperation() + + val warmUpIterations = if (fast) 0 else 5 + val iterations = if (fast) 1 else 5 + + var lastProject: Project? = null + var counter = 0 + + performanceTest { + name("open project $name${if (note.isNotEmpty()) " $note" else ""}") + stats(stats) + warmUpIterations(warmUpIterations) + iterations(iterations) + checkStability(!fast) + test { + it.value = openProjectOperation.openProject() + } + tearDown { + it.value?.let { project -> + lastProject = project + openProjectOperation.postOpenProject(project) + + logMessage { "project '$name' successfully opened" } + + // close all project but last - we're going to return and use it further + if (counter < warmUpIterations + iterations - 1) { + closeProject(project) + } + counter++ + } + } + } + + // indexing + lastProject?.let { project -> + invalidateLibraryCache(project) + + CodeInsightTestFixtureImpl.ensureIndexesUpToDate(project) + + dispatchAllInvocationEvents() + + logMessage { "project $name is ${if (project.isInitialized) "initialized" else "not initialized"}" } + + with(DumbService.getInstance(project)) { + queueTask(UnindexedFilesUpdater(project)) + completeJustSubmittedTasks() + } + dispatchAllInvocationEvents() + + Fixture.enableAnnotatorsAndLoadDefinitions(project) + + myApplication.setDataProvider(TestDataProvider(project)) + } + + return lastProject ?: error("unable to open project $name") + } protected fun perfOpenProject( name: String, @@ -170,16 +197,13 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { it.value?.let { project -> lastProject = project openAction.postOpenProject(openProject = openProject, project = project) - openAction.initDefaultProfile(project) + project.initDefaultProfile() logMessage { "project '$name' successfully opened" } // close all project but last - we're going to return and use it further if (counter < warmUpIterations + iterations - 1) { - myApplication.setDataProvider(null) - logMessage { "project '$name' is about to be closed" } - closeProject(project) - logMessage { "project '$name' successfully closed" } + myApplication.closeProject(project) } counter++ } @@ -448,92 +472,6 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { } } - fun perfType( - stats: Stats, - fileName: String, - marker: String, - insertString: String, - surroundItems: String = "\n", - typeAfterMarker: Boolean = true, - revertChangesAtTheEnd: Boolean = true, - note: String = "", - delay: Long? = null - ) = perfType( - project(), stats, fileName, marker, insertString, surroundItems, - typeAfterMarker = typeAfterMarker, - revertChangesAtTheEnd = revertChangesAtTheEnd, - note = note, - delay = delay - ) - - fun perfType( - project: Project, - stats: Stats, - fileName: String, - marker: String, - insertString: String, - surroundItems: String = "\n", - typeAfterMarker: Boolean = true, - revertChangesAtTheEnd: Boolean = true, - note: String = "", - delay: Long? = null - ) { - performanceTest, List> { - name("type ${notePrefix(note)}$fileName") - stats(stats) - warmUpIterations(8) - iterations(15) - setUp { - val fixture = openFixture(project, fileName) - val editor = fixture.editor - - val initialText = editor.document.text - updateScriptDependenciesIfNeeded(fileName, fixture) - - val tasksIdx = editor.document.text.indexOf(marker) - assertTrue("marker '$marker' not found in $fileName", tasksIdx > 0) - if (typeAfterMarker) { - editor.caretModel.moveToOffset(tasksIdx + marker.length + 1) - } else { - editor.caretModel.moveToOffset(tasksIdx - 1) - } - - for (surroundItem in surroundItems) { - EditorTestUtil.performTypingAction(editor, surroundItem) - } - - editor.caretModel.moveToOffset(editor.caretModel.offset - if (typeAfterMarker) 1 else 2) - - if (!typeAfterMarker) { - for (surroundItem in surroundItems) { - EditorTestUtil.performTypingAction(editor, surroundItem) - } - editor.caretModel.moveToOffset(editor.caretModel.offset - 2) - } - - it.setUpValue = Pair(initialText, fixture) - } - test { - val fixture = it.setUpValue!!.second - for (i in insertString.indices) { - fixture.type(insertString[i]) - delay?.let { d -> Thread.sleep(d) } - } - it.value = fixture.doHighlighting() - } - tearDown { - it.value?.let { list -> - assertNotEmpty(list) - } - it.setUpValue?.let { pair -> - pair.second.revertChanges(revertChangesAtTheEnd, pair.first) - } - commitAllDocuments() - } - profilerEnabled(true) - } - } - fun perfCopyAndPaste( stats: Stats, sourceFileName: String, @@ -630,12 +568,16 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { protected fun perfHighlightFileEmptyProfile(name: String, stats: Stats): List = perfHighlightFile(project(), name, stats, tools = emptyArray(), note = "empty profile") - private fun perfHighlightFile( + protected fun perfHighlightFile( project: Project, fileName: String, stats: Stats, tools: Array? = null, - note: String = "" + note: String = "", + warmUpIterations: Int = 3, + iterations: Int = 10, + checkStability: Boolean = true, + filenameSimplifier: (String) -> String = ::simpleFilename ): List { val profileManager = ProjectInspectionProfileManager.getInstance(project) val currentProfile = profileManager.currentProfile @@ -647,10 +589,11 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { val isWarmUp = note == WARM_UP var highlightInfos: List = emptyList() performanceTest> { - name("highlighting ${notePrefix(note)}${simpleFilename(fileName)}") + name("highlighting ${notePrefix(note)}${filenameSimplifier(fileName)}") stats(stats) - warmUpIterations(if (isWarmUp) 1 else 3) - iterations(if (isWarmUp) 2 else 10) + warmUpIterations(if (isWarmUp) 1 else warmUpIterations) + iterations(if (isWarmUp) 2 else iterations) + checkStability(checkStability) setUp { it.setUpValue = openFileInEditor(project, fileName) } @@ -661,7 +604,10 @@ abstract class AbstractPerformanceProjectsTest : UsefulTestCase() { tearDown { highlightInfos = it.value ?: emptyList() commitAllDocuments() - FileEditorManager.getInstance(project).closeFile(it.setUpValue!!.psiFile.virtualFile) + it.setUpValue?.let { editorFile -> + val fileEditorManager = FileEditorManager.getInstance(project) + fileEditorManager.closeFile(editorFile.psiFile.virtualFile) + } PsiManager.getInstance(project).dropPsiCaches() } profilerEnabled(!isWarmUp) diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/FileUtils.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/FileUtils.kt deleted file mode 100644 index 9a39f5add70..00000000000 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/FileUtils.kt +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.idea.perf - -import java.nio.file.* - -fun Path.copyRecursively(targetDirectory: Path) { - val src = this - Files.walk(src) - .forEach { source -> Files.copy(source, targetDirectory.resolve(src.relativize(source)), StandardCopyOption.REPLACE_EXISTING) } -} \ No newline at end of file diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceNativeProjectsTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceNativeProjectsTest.kt index 9ae3f482f8a..575df0db361 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceNativeProjectsTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceNativeProjectsTest.kt @@ -25,9 +25,9 @@ import org.jetbrains.kotlin.idea.framework.detectLibraryKind import org.jetbrains.kotlin.idea.perf.PerformanceNativeProjectsTest.TestProject.* import org.jetbrains.kotlin.idea.perf.PerformanceNativeProjectsTest.TestTarget.* import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP -import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcSuite +import org.jetbrains.kotlin.idea.perf.util.TeamCity.suite +import org.jetbrains.kotlin.idea.perf.util.logMessage import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction.GRADLE_PROJECT -import org.jetbrains.kotlin.idea.testFramework.logMessage import org.jetbrains.kotlin.idea.testFramework.suggestOsNeutralFileName import org.jetbrains.kotlin.idea.util.projectStructure.allModules import org.jetbrains.kotlin.library.KOTLIN_STDLIB_NAME @@ -157,7 +157,7 @@ class PerformanceNativeProjectsTest : AbstractPerformanceProjectsTest() { assertTrue("Target $testTarget is not allowed on your host OS", testTarget.enabled) val projectName = projectName(testTarget, testProject, enableCommonizer) - tcSuite(projectName) { + suite(projectName) { Stats(projectName).use { stats -> myProject = perfOpenTemplateGradleProject(stats, testTarget, testProject, enableCommonizer) diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt index 7335dadb3fc..653d1035b18 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt @@ -8,15 +8,13 @@ package org.jetbrains.kotlin.idea.perf import com.intellij.codeInsight.daemon.impl.HighlightInfo import com.intellij.lang.annotation.AnnotationHolder import com.intellij.openapi.project.Project -import com.intellij.openapi.util.Disposer import com.intellij.psi.PsiElement import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager import org.jetbrains.kotlin.idea.highlighter.KotlinPsiChecker import org.jetbrains.kotlin.idea.highlighter.KotlinPsiCheckerAndHighlightingUpdater import org.jetbrains.kotlin.idea.perf.Stats.Companion.TEST_KEY -import org.jetbrains.kotlin.idea.perf.Stats.Companion.WARM_UP import org.jetbrains.kotlin.idea.perf.Stats.Companion.runAndMeasure -import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcSuite +import org.jetbrains.kotlin.idea.perf.util.TeamCity.suite import org.jetbrains.kotlin.idea.testFramework.Fixture import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.cleanupCaches import org.jetbrains.kotlin.idea.testFramework.Fixture.Companion.isAKotlinScriptFile @@ -29,10 +27,10 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { companion object { @JvmStatic - var warmedUp: Boolean = false + val hwStats: Stats = Stats("helloWorld project") @JvmStatic - val hwStats: Stats = Stats("helloWorld project") + val warmUp = WarmUpProject(hwStats) @JvmStatic val timer: AtomicLong = AtomicLong() @@ -53,17 +51,28 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { override fun setUp() { super.setUp() - // warm up: open simple small project - if (!warmedUp) { - warmUpProject(hwStats, "src/HelloMain.kt") { perfOpenHelloWorld(hwStats, WARM_UP) } - warmedUp = true - } + warmUp.warmUp(this) } fun testHelloWorldProject() { + suite("Hello world project") { + myProject = perfOpenProject(stats = hwStats) { + name("helloKotlin") - tcSuite("Hello world project") { - myProject = perfOpenHelloWorld(hwStats) + kotlinFile("HelloMain") { + topFunction("main") { + param("args", "Array") + body("""println("Hello World!")""") + } + } + + kotlinFile("HelloMain2") { + topFunction("main") { + param("args", "Array") + body("""println("Hello World!")""") + } + } + } // highlight perfHighlightFile("src/HelloMain.kt", hwStats) @@ -72,9 +81,8 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { } fun testKotlinProject() { - tcSuite("Kotlin project") { - val stats = Stats("kotlin project") - stats.use { + suite("Kotlin project") { + Stats("kotlin project").use { perfOpenKotlinProject(it) val filesToHighlight = arrayOf( @@ -114,9 +122,8 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { } fun testKotlinProjectCopyAndPaste() { - tcSuite("Kotlin copy-and-paste") { - val stats = Stats("Kotlin copy-and-paste") - stats.use { stat -> + suite("Kotlin copy-and-paste") { + Stats("Kotlin copy-and-paste").use { stat -> perfOpenKotlinProjectFast(stat) perfCopyAndPaste( @@ -129,9 +136,8 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { } fun testKotlinProjectCompletionKtFile() { - tcSuite("Kotlin completion ktFile") { - val stats = Stats("Kotlin completion ktFile") - stats.use { stat -> + suite("Kotlin completion ktFile") { + Stats("Kotlin completion ktFile").use { stat -> perfOpenKotlinProjectFast(stat) perfTypeAndAutocomplete( @@ -157,9 +163,8 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { } fun testKotlinProjectCompletionBuildGradle() { - tcSuite("Kotlin completion gradle.kts") { - val stats = Stats("kotlin completion gradle.kts") - stats.use { stat -> + suite("Kotlin completion gradle.kts") { + Stats("kotlin completion gradle.kts").use { stat -> runAndMeasure("open kotlin project") { perfOpenKotlinProjectFast(stat) } @@ -191,9 +196,8 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { } fun testKotlinProjectScriptDependenciesBuildGradle() { - tcSuite("Kotlin scriptDependencies gradle.kts") { - val stats = Stats("kotlin scriptDependencies gradle.kts") - stats.use { stat -> + suite("Kotlin scriptDependencies gradle.kts") { + Stats("kotlin scriptDependencies gradle.kts").use { stat -> perfOpenKotlinProjectFast(stat) perfScriptDependenciesBuildGradleKts(stat) @@ -205,9 +209,8 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { } fun testKotlinProjectBuildGradle() { - tcSuite("Kotlin gradle.kts") { - val stats = Stats("kotlin gradle.kts") - stats.use { stat -> + suite("Kotlin gradle.kts") { + Stats("kotlin gradle.kts").use { stat -> perfOpenKotlinProjectFast(stat) perfFileAnalysisBuildGradleKts(stat) @@ -280,14 +283,14 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { val extraStats = Stats("${stats.name} $testName") val extraTimingsNs = mutableListOf?>() - val warmUpIterations = 20 - val iterations = 30 + val warmUpIterations = 30 + val iterations = 50 performanceTest>> { name(testName) stats(stats) - warmUpIterations(30) - iterations(50) + warmUpIterations(warmUpIterations) + iterations(iterations) setUp(perfKtsFileAnalysisSetUp(project, fileName)) test(perfKtsFileAnalysisTest()) tearDown(perfKtsFileAnalysisTearDown(extraTimingsNs, project)) diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceStressTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceStressTest.kt index 90ff87daaac..4ddc5a3f52c 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceStressTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/PerformanceStressTest.kt @@ -5,95 +5,91 @@ package org.jetbrains.kotlin.idea.perf -import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcSuite +import com.intellij.codeInsight.daemon.impl.HighlightInfo +import com.intellij.testFramework.UsefulTestCase +import org.jetbrains.kotlin.idea.perf.util.DefaultProfile +import org.jetbrains.kotlin.idea.perf.util.PerformanceSuite +import org.jetbrains.kotlin.idea.perf.util.PerformanceSuite.TypingConfig +import org.jetbrains.kotlin.idea.perf.util.ProjectProfile +import org.jetbrains.kotlin.idea.perf.util.suite +import org.jetbrains.kotlin.idea.testFramework.commitAllDocuments +import org.jetbrains.kotlin.test.JUnit3RunnerWithInners +import org.junit.runner.RunWith -class PerformanceStressTest : AbstractPerformanceProjectsTest() { - - companion object { - - @JvmStatic - var warmedUp: Boolean = false - - @JvmStatic - val hwStats: Stats = Stats("helloWorld project") - - init { - // there is no @AfterClass for junit3.8 - Runtime.getRuntime().addShutdownHook(Thread { hwStats.close() }) - } - - } - - override fun setUp() { - super.setUp() - // warm up: open simple small project - if (!warmedUp) { - warmUpProject(hwStats, "src/HelloMain.kt") { - openProject { - name("helloWorld") - - kotlinFile("HelloMain") { - topFunction("main") { - param("args", "Array") - body("""println("Hello World!")""") - } - } - } - } - warmedUp = true - } - } +@RunWith(JUnit3RunnerWithInners::class) +class PerformanceStressTest : UsefulTestCase() { fun testLotsOfOverloadedMethods() { // KT-35135 val generatedTypes = mutableListOf(listOf()) generateTypes(arrayOf("Int", "String", "Long", "List", "Array"), generatedTypes) - tcSuite("Lots of overloaded method project") { - myProject = openProject { - name("kt-35135") - buildGradle("idea/testData/perfTest/simpleTemplate/") + suite( + suiteName = "Lots of overloaded method project", + config = PerformanceSuite.StatsScopeConfig(name = "kt-35135 project", warmup = 8, iterations = 15) + ) { + app { + warmUpProject() - kotlinFile("OverloadX") { - pkg("pkg") + project { + descriptor { + name("kt-35135") + buildGradle("idea/testData/perfTest/simpleTemplate/") - topClass("OverloadX") { - openClass() + kotlinFile("OverloadX") { + pkg("pkg") - for (types in generatedTypes) { - function("foo") { - openFunction() - returnType("String") - for ((index, type) in types.withIndex()) { - param("arg$index", type) + topClass("OverloadX") { + openClass() + + for (types in generatedTypes) { + function("foo") { + openFunction() + returnType("String") + for ((index, type) in types.withIndex()) { + param("arg$index", type) + } + body("TODO()") + } } - body("TODO()") + } + } + + kotlinFile("SomeClass") { + pkg("pkg") + + topClass("SomeClass") { + superClass("OverloadX") + + body("ov") + } + } + } + + profile(DefaultProfile) + + fixture("src/main/java/pkg/SomeClass.kt").use { fixture -> + val typingConfig = TypingConfig( + fixture, + marker = "ov", + insertString = "override fun foo(): String = TODO()", + delayMs = 50 + ) + + measure>("type override fun foo()", fixture = fixture) { + before = { + moveCursor(typingConfig) + } + test = { + typeAndHighlight(typingConfig) + } + after = { + fixture.restoreText() + commitAllDocuments() } } } } - - kotlinFile("SomeClass") { - pkg("pkg") - - topClass("SomeClass") { - superClass("OverloadX") - - body("ov") - } - } - } - - val stats = Stats("kt-35135 project") - stats.use { stats -> - perfType( - stats, - "src/main/java/pkg/SomeClass.kt", - "ov", - "override fun foo(): String = TODO()", - note = "override fun foo()", - delay = 50 - ) } } } diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/ProjectBuilder.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/ProjectBuilder.kt index 48a41ad08bb..d8c0c42f65e 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/ProjectBuilder.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/ProjectBuilder.kt @@ -9,6 +9,8 @@ import com.intellij.openapi.project.Project import com.intellij.openapi.projectRoots.JavaSdk import com.intellij.openapi.projectRoots.impl.JavaAwareProjectJdkTableImpl import com.intellij.util.io.* +import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.initDefaultProfile +import org.jetbrains.kotlin.idea.testFramework.GRADLE_JDK_NAME import org.jetbrains.kotlin.idea.testFramework.OpenProject import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction import java.nio.file.Files @@ -209,31 +211,54 @@ class ProjectBuilder { // return targetDirectory.toAbsolutePath().toFile().absolutePath } + + fun openProjectOperation(): OpenProjectOperation { + val builder = this + val projectPath = generateFiles() + + val jdkTableImpl = JavaAwareProjectJdkTableImpl.getInstanceEx() + val homePath = if (jdkTableImpl.internalJdk.homeDirectory!!.name == "jre") { + jdkTableImpl.internalJdk.homeDirectory!!.parent.path + } else { + jdkTableImpl.internalJdk.homePath!! + } + val javaSdk = JavaSdk.getInstance() + val jdk18 = javaSdk.createJdk("1.8", homePath) + + val openAction = if (buildGradle != null) ProjectOpenAction.GRADLE_PROJECT else ProjectOpenAction.SIMPLE_JAVA_MODULE + val openProject = OpenProject( + projectPath = projectPath, + projectName = name, + jdk = jdk18, + projectOpenAction = openAction + ) + + return object : OpenProjectOperation { + + override fun openProject() = ProjectOpenAction.openProject(openProject) + + override fun postOpenProject(project: Project) { + openAction.postOpenProject(project = project, openProject = openProject) + if (builder.initDefaultProfile) { + project.initDefaultProfile() + } + } + + } + } +} + +interface OpenProjectOperation { + fun openProject(): Project + + fun postOpenProject(project: Project) } fun openProject(initializer: ProjectBuilder.() -> Unit): Project { val projectBuilder = ProjectBuilder().apply(initializer) - val projectPath = projectBuilder.generateFiles() + val openProject = projectBuilder.openProjectOperation() - val jdkTableImpl = JavaAwareProjectJdkTableImpl.getInstanceEx() - val homePath = if (jdkTableImpl.internalJdk.homeDirectory!!.name == "jre") { - jdkTableImpl.internalJdk.homeDirectory!!.parent.path - } else { - jdkTableImpl.internalJdk.homePath!! + return openProject.openProject().also { + openProject.postOpenProject(it) } - val javaSdk = JavaSdk.getInstance() - val jdk18 = javaSdk.createJdk("1.8", homePath) - val openAction = if (projectBuilder.buildGradle != null) ProjectOpenAction.GRADLE_PROJECT else ProjectOpenAction.SIMPLE_JAVA_MODULE - val openProject = OpenProject( - projectPath = projectPath, - projectName = projectBuilder.name, - jdk = jdk18, - projectOpenAction = openAction - ) - val project = ProjectOpenAction.openProject(openProject) - openAction.postOpenProject(project = project, openProject = openProject) - if (projectBuilder.initDefaultProfile) { - openAction.initDefaultProfile(project) - } - return project } \ No newline at end of file diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt index 1d6a1054978..3163ee20d2c 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/Stats.kt @@ -7,7 +7,8 @@ package org.jetbrains.kotlin.idea.perf import org.jetbrains.kotlin.idea.perf.WholeProjectPerformanceTest.Companion.nsToMs import org.jetbrains.kotlin.idea.perf.profilers.* -import org.jetbrains.kotlin.idea.testFramework.logMessage +import org.jetbrains.kotlin.idea.perf.util.TeamCity +import org.jetbrains.kotlin.idea.perf.util.logMessage import org.jetbrains.kotlin.idea.testFramework.suggestOsNeutralFileName import org.jetbrains.kotlin.util.PerformanceCounter import java.io.* @@ -57,9 +58,9 @@ class Stats( )) { val n = "$id : ${v.first}" - printTestStarted(n) - printStatValue("$id${v.second}", v.third) - printTestFinished(n, v.third, includeStatValue = false) + TeamCity.test(n, durationMs = v.third) { + TeamCity.statValue("$id${v.second}", v.third) + } } statInfosArray.filterNotNull() @@ -75,9 +76,11 @@ class Stats( val shortName = if (perfCounterName.endsWith(": time")) n.removeSuffix(": time") else null - shortName?.let { printTestStarted(it) } - printStatValue(n, mean) - shortName?.let { printTestFinished(it, mean, includeStatValue = false) } + shortName?.let { + TeamCity.test(it, durationMs = mean) { + TeamCity.statValue(it, mean) + } + } } perfTestRawDataMs.addAll(timingsMs.toList()) @@ -161,15 +164,15 @@ class Stats( val stabilityName = "$name: $testName stability" val stable = stabilityPercentage <= acceptanceStabilityLevel - printTestStarted(stabilityName) - printStatValue(stabilityName, stabilityPercentage) - if (stable or !checkStability) { - printTestFinished(stabilityName) + + val error = if (stable or !checkStability) { + null } else { - printTestFailed( - stabilityName, - "$testName stability is $stabilityPercentage %, above accepted level of $acceptanceStabilityLevel %" - ) + "$testName stability is $stabilityPercentage %, above accepted level of $acceptanceStabilityLevel %" + } + + TeamCity.test(stabilityName, errorDetails = error) { + TeamCity.statValue(stabilityName, stabilityPercentage) } } } else { @@ -178,7 +181,7 @@ class Stats( } if (testName != WARM_UP) { - tcSuite(testName, block) + TeamCity.suite(testName, block) } else { block() } @@ -197,19 +200,17 @@ class Stats( val t = statInfo[ERROR_KEY] as? Throwable if (t != null) { - printTestStarted(n) - printTestFinished(n, t) + TeamCity.test(n, errors = listOf(t)) {} } else if (!printOnlyErrors) { - printTestStarted(n) - for ((k, v) in statInfo) { - if (k == TEST_KEY) continue - printStatValue("$n $k", v) - (v as? Number)?.let { - printTestMetadata(n, k, it) + TeamCity.test(n, durationMs = (statInfo[TEST_KEY] as Long).nsToMs) { + for ((k, v) in statInfo) { + if (k == TEST_KEY) continue + TeamCity.statValue("$n $k", v) + (v as? Number)?.let { + TeamCity.testMetadata(n, k, it) + } } } - - printTestFinished(n, (statInfo[TEST_KEY] as Long).nsToMs) } } } @@ -245,7 +246,14 @@ class Stats( private fun mainPhase(phaseData: PhaseData): Array { val statInfosArray = phase(phaseData, "") - statInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let { throw it } + statInfosArray.filterNotNull().map { it[ERROR_KEY] as? Throwable }.firstOrNull()?.let { + printTimings( + phaseData.testName, + printOnlyErrors = true, + statInfoArray = statInfosArray + ) + throw it + } return statInfosArray } @@ -300,7 +308,7 @@ class Stats( } } catch (t: Throwable) { logMessage(t) { "error at ${phaseData.testName}" } - tcPrintErrors(phaseData.testName, listOf(t)) + TeamCity.testFailed(name, error = t) } return statInfosArray } @@ -338,7 +346,7 @@ class Stats( override fun close() { if (perfTestRawDataMs.isNotEmpty()) { val geomMeanMs = geomMean(perfTestRawDataMs.toList()).toLong() - printStatValue("$name geomMean", geomMeanMs) + TeamCity.statValue("$name geomMean", geomMeanMs) append(arrayOf("$name geomMean", geomMeanMs, 0)) } statsOutput.flush() @@ -357,75 +365,6 @@ class Stats( } logMessage { "$note took $openProjectMillis ms" } } - - fun printTestStarted(testName: String) { - println("##teamcity[testStarted name='$testName' captureStandardOutput='true']") - } - - fun printStatValue(name: String, value: Any) { - println("##teamcity[buildStatisticValue key='$name' value='$value']") - } - - fun printTestMetadata(testName: String, name: String, value: Number) { - println("##teamcity[testMetadata testName='$testName' name='$name' type='number' value='$value']") - } - - private fun printTestFinished(testName: String) { - println("##teamcity[testFinished name='$testName']") - } - - fun printTestFinished(testName: String, spentMs: Long, includeStatValue: Boolean = true) { - if (includeStatValue) { - printStatValue(testName, spentMs) - } - println("##teamcity[testFinished name='$testName' duration='$spentMs']") - } - - fun printTestFinished(testName: String, error: Throwable) { - println("error at $testName:") - printStatValue(testName, -1) - tcPrintErrors(testName, listOf(error)) - } - - private fun printTestFailed(testName: String, details: String) { - println("##teamcity[testFailed name='$testName' message='Exceptions reported' details='${tcEscape(details)}']") - } - - inline fun tcSuite(name: String, block: () -> Unit) { - println("##teamcity[testSuiteStarted name='$name']") - try { - block() - } finally { - println("##teamcity[testSuiteFinished name='$name']") - } - } - - inline fun tcTest(name: String, block: () -> Pair>) { - printTestStarted(name) - val (time, errors) = block() - tcPrintErrors(name, errors) - } - - private fun tcEscape(s: String) = s.replace("|", "||") - .replace("[", "|[") - .replace("]", "|]") - .replace("\r", "|r") - .replace("\n", "|n") - .replace("'", "|'") - - fun tcPrintErrors(name: String, errors: List) { - if (errors.isNotEmpty()) { - val detailsWriter = StringWriter() - val errorDetailsPrintWriter = PrintWriter(detailsWriter) - errors.forEach { - it.printStackTrace(errorDetailsPrintWriter) - errorDetailsPrintWriter.println() - } - errorDetailsPrintWriter.close() - val details = detailsWriter.toString() - printTestFailed(name, details) - } - } } } diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WarmUpProject.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WarmUpProject.kt new file mode 100644 index 00000000000..802e9bb491a --- /dev/null +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WarmUpProject.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.idea.perf + +/** + * warm up: open simple `hello world` project + */ +class WarmUpProject(private val stats: Stats) { + private var warmedUp: Boolean = false + + fun warmUp(test: AbstractPerformanceProjectsTest) { + if (warmedUp) return + test.warmUpProject(stats, "src/HelloMain.kt") { + openProject { + name("helloWorld") + + kotlinFile("HelloMain") { + topFunction("main") { + param("args", "Array") + body("""println("Hello World!")""") + } + } + } + } + warmedUp = true + } +} \ No newline at end of file diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WholeProjectPerformanceTest.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WholeProjectPerformanceTest.kt index 94d5a3b91c9..3f82ff8440a 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WholeProjectPerformanceTest.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/WholeProjectPerformanceTest.kt @@ -28,8 +28,8 @@ import com.intellij.psi.impl.source.resolve.reference.ReferenceProvidersRegistry import com.intellij.psi.xml.XmlFileNSInfoProvider import com.intellij.xml.XmlSchemaProvider import org.jetbrains.kotlin.idea.framework.KotlinSdkType -import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcSuite -import org.jetbrains.kotlin.idea.perf.Stats.Companion.tcTest +import org.jetbrains.kotlin.idea.perf.util.TeamCity +import org.jetbrains.kotlin.idea.testFramework.TestApplicationManager import org.jetbrains.kotlin.idea.util.getProjectJdkTableSafe import java.io.File @@ -95,7 +95,7 @@ abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProj fun testWholeProjectPerformance() { - tcSuite(this::class.simpleName ?: "Unknown") { + TeamCity.suite(this::class.simpleName ?: "Unknown") { val totals = mutableMapOf() fun appendInspectionResult(file: String, id: String, nanoTime: Long) { @@ -104,7 +104,7 @@ abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProj perfStats.append(file, id, nanoTime) } - tcSuite("TotalPerFile") { + TeamCity.suite("TotalPerFile") { // TODO: [VD] temp to limit number of files (and total time to run) val files = provideFiles(project).sortedBy { it.name }.take(10) @@ -120,7 +120,7 @@ abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProj } } - tcSuite("Total") { + TeamCity.suite("Total") { totals.forEach { (k, v) -> tcTest(k) { v.nsToMs to emptyList() @@ -139,6 +139,11 @@ abstract class WholeProjectPerformanceTest : DaemonAnalyzerTestCase(), WholeProj companion object { val Long.nsToMs get() = (this * 1e-6).toLong() + + inline fun tcTest(name: String, block: () -> Pair>) { + val (time, errors) = block() + TeamCity.test(name, time, errors = errors) {} + } } } \ No newline at end of file diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/compat.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/compat.kt deleted file mode 100644 index 1f21d3bdde1..00000000000 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/compat.kt +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -@file:Suppress("UNUSED_PARAMETER") - -package org.jetbrains.kotlin.idea.perf - -import com.intellij.openapi.Disposable -import com.intellij.openapi.project.Project -import com.intellij.testFramework.propertyBased.MadTestingUtil - -// BUNCH: 193 -fun enableAllInspectionsCompat(project: Project, disposable: Disposable) { - MadTestingUtil.enableAllInspections(project, disposable) -} - -// BUNCH: 193 -typealias TestApplicationManager = com.intellij.idea.IdeaTestApplication diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/compat.kt.201 b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/compat.kt.201 deleted file mode 100644 index 169f9061dc1..00000000000 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/compat.kt.201 +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -@file:Suppress("UNUSED_PARAMETER") - -package org.jetbrains.kotlin.idea.perf - -import com.intellij.openapi.Disposable -import com.intellij.openapi.project.Project -import com.intellij.testFramework.propertyBased.MadTestingUtil - -// BUNCH: 193 -fun enableAllInspectionsCompat(project: Project, disposable: Disposable) { - MadTestingUtil.enableAllInspections(project) -} - -// BUNCH: 193 -typealias TestApplicationManager = com.intellij.testFramework.TestApplicationManager \ No newline at end of file diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/ProfilerHandler.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/ProfilerHandler.kt index 160bfc1331a..cc899413983 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/ProfilerHandler.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/ProfilerHandler.kt @@ -7,7 +7,7 @@ package org.jetbrains.kotlin.idea.perf.profilers import org.jetbrains.kotlin.idea.perf.profilers.async.AsyncProfilerHandler import org.jetbrains.kotlin.idea.perf.profilers.yk.YKProfilerHandler -import org.jetbrains.kotlin.idea.testFramework.logMessage +import org.jetbrains.kotlin.idea.perf.util.logMessage interface ProfilerHandler { fun startProfiling(activityName: String, options: List = emptyList()) diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/async/AsyncProfilerHandler.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/async/AsyncProfilerHandler.kt index ea2fe0b63f5..0b994753fdb 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/async/AsyncProfilerHandler.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/async/AsyncProfilerHandler.kt @@ -10,7 +10,7 @@ import com.intellij.openapi.util.io.FileUtil import com.intellij.util.containers.ContainerUtil import org.jetbrains.kotlin.idea.perf.profilers.ProfilerHandler import org.jetbrains.kotlin.idea.perf.profilers.doOrThrow -import org.jetbrains.kotlin.idea.testFramework.logMessage +import org.jetbrains.kotlin.idea.perf.util.logMessage import java.io.File import java.lang.reflect.Method import java.nio.file.Files diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/yk/YKProfilerHandler.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/yk/YKProfilerHandler.kt index 15cfaf946c2..1061509d6aa 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/yk/YKProfilerHandler.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/profilers/yk/YKProfilerHandler.kt @@ -7,7 +7,7 @@ package org.jetbrains.kotlin.idea.perf.profilers.yk import org.jetbrains.kotlin.idea.perf.profilers.ProfilerHandler import org.jetbrains.kotlin.idea.perf.profilers.doOrThrow -import org.jetbrains.kotlin.idea.testFramework.logMessage +import org.jetbrains.kotlin.idea.perf.util.logMessage import java.lang.management.ManagementFactory import java.lang.reflect.Method import java.nio.file.Files diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/fileTools.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/fileTools.kt new file mode 100644 index 00000000000..950812fa4c6 --- /dev/null +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/fileTools.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.idea.perf.util + +import com.intellij.util.io.exists +import java.io.File +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.StandardCopyOption + +fun String.lastPathSegment() = + Paths.get(this).last().toString() + +fun exists(path: String, vararg paths: String) = + Paths.get(path, *paths).toAbsolutePath().exists() + +fun Path.copyRecursively(targetDirectory: Path) { + val src = this + Files.walk(src) + .forEach { source -> Files.copy(source, targetDirectory.resolve(src.relativize(source)), StandardCopyOption.REPLACE_EXISTING) } +} + +fun File.allFilesWithExtension(ext: String): Iterable = + walk().filter { it.isFile && it.extension.equals(ext, ignoreCase = true) }.toList() diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/logging.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/logging.kt new file mode 100644 index 00000000000..c94eedd6dfd --- /dev/null +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/logging.kt @@ -0,0 +1,99 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.idea.perf.util + +import java.io.PrintWriter +import java.io.StringWriter + +inline fun gradleMessage(block: () -> String) { + print("#gradle ${block()}") +} + +inline fun logMessage(message: () -> String) { + println("-- ${message()}") +} + +fun logMessage(t: Throwable, message: () -> String) { + val writer = StringWriter() + PrintWriter(writer).use { + t.printStackTrace(it) + } + println("-- ${message()}:\n$writer") +} + +object TeamCity { + inline fun message(block: () -> String) { + println("##teamcity[${block()}]") + } + + fun suite(name: String, block: () -> Unit) { + message { "testSuiteStarted name='$name'" } + try { + block() + } finally { + message { "testSuiteFinished name='$name'" } + } + } + + fun test(name: String, durationMs: Long? = null, errors: List, block: () -> Unit) { + test(name, durationMs, if (errors.isNotEmpty()) toDetails(errors) else null, block) + } + + fun test(name: String, durationMs: Long? = null, errorDetails: String? = null, block: () -> Unit) { + testStarted(name) + try { + block() + } finally { + if (errorDetails != null) { + statValue(name, -1) + testFailed(name, errorDetails) + } else { + testFinished(name, durationMs) + } + } + } + + inline fun statValue(name: String, value: Any) { + message { "buildStatisticValue key='$name' value='$value'" } + } + + inline fun testStarted(testName: String) { + message { "testStarted name='$testName'" } + } + + inline fun testMetadata(testName: String, name: String, value: Number) { + message { "testMetadata testName='$testName' name='$name' type='number' value='$value'" } + } + + inline fun testFinished(testName: String, durationMs: Long? = null) { + message { "testFinished name='$testName'${durationMs?.let { " duration='$durationMs'" }}" } + } + + fun testFailed(testName: String, error: Throwable) = testFailed(testName, toDetails(listOf(error))!!) + + fun testFailed(testName: String, details: String) { + message { "testFailed name='$testName' message='Exceptions reported' details='${escape(details)}'" } + } + + private fun escape(s: String) = s.replace("|", "||") + .replace("[", "|[") + .replace("]", "|]") + .replace("\r", "|r") + .replace("\n", "|n") + .replace("'", "|'") + + private fun toDetails(errors: List): String? { + if (errors.isEmpty()) return null + val detailsWriter = StringWriter() + val errorDetailsPrintWriter = PrintWriter(detailsWriter) + errors.forEach { + it.printStackTrace(errorDetailsPrintWriter) + errorDetailsPrintWriter.println() + } + errorDetailsPrintWriter.close() + return detailsWriter.toString() + } +} \ No newline at end of file diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/performanceSuite.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/performanceSuite.kt new file mode 100644 index 00000000000..7e4325bef03 --- /dev/null +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/performanceSuite.kt @@ -0,0 +1,416 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.idea.perf.util + +import com.intellij.codeInsight.daemon.impl.HighlightInfo +import com.intellij.openapi.Disposable +import com.intellij.openapi.application.runWriteAction +import com.intellij.openapi.editor.EditorFactory +import com.intellij.openapi.fileEditor.FileDocumentManager +import com.intellij.openapi.fileEditor.FileEditorManager +import com.intellij.openapi.project.DumbService +import com.intellij.openapi.project.Project +import com.intellij.openapi.project.ex.ProjectManagerEx +import com.intellij.openapi.projectRoots.JavaSdk +import com.intellij.openapi.projectRoots.Sdk +import com.intellij.openapi.projectRoots.impl.JavaAwareProjectJdkTableImpl +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.psi.PsiDocumentManager +import com.intellij.psi.PsiFile +import com.intellij.psi.PsiManager +import com.intellij.psi.impl.PsiDocumentManagerBase +import com.intellij.testFramework.* +import com.intellij.testFramework.fixtures.impl.CodeInsightTestFixtureImpl +import com.intellij.util.ArrayUtilRt +import com.intellij.util.ThrowableRunnable +import com.intellij.util.containers.toArray +import com.intellij.util.indexing.UnindexedFilesUpdater +import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager +import org.jetbrains.kotlin.idea.framework.KotlinSdkType +import org.jetbrains.kotlin.idea.perf.ProjectBuilder +import org.jetbrains.kotlin.idea.perf.Stats +import org.jetbrains.kotlin.idea.perf.Stats.Companion.runAndMeasure +import org.jetbrains.kotlin.idea.perf.performanceTest +import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.disableAllInspections +import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.enableAllInspections +import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.enableInspections +import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.enableSingleInspection +import org.jetbrains.kotlin.idea.perf.util.ProfileTools.Companion.initDefaultProfile +import org.jetbrains.kotlin.idea.test.invalidateLibraryCache +import org.jetbrains.kotlin.idea.testFramework.* +import org.jetbrains.kotlin.idea.testFramework.TestApplicationManager +import org.jetbrains.kotlin.idea.util.getProjectJdkTableSafe +import java.io.File + +/** + * @author Vladimir Ilmov + */ +class PerformanceSuite { + companion object { + + fun suite( + name: String, + stats: StatsScope, + block: (StatsScope) -> Unit + ) { + TeamCity.suite(name) { + stats.stats.use { + block(stats) + } + } + } + + private fun PsiFile.highlightFile(): List { + val document = FileDocumentManager.getInstance().getDocument(virtualFile)!! + val editor = EditorFactory.getInstance().getEditors(document).first() + PsiDocumentManager.getInstance(project).commitAllDocuments() + return CodeInsightTestFixtureImpl.instantiateAndRun(this, editor, ArrayUtilRt.EMPTY_INT_ARRAY, true) + } + + fun rollbackChanges(vararg file: VirtualFile) { + val fileDocumentManager = FileDocumentManager.getInstance() + runInEdtAndWait { + fileDocumentManager.reloadFiles(*file) + } + + ProjectManagerEx.getInstanceEx().openProjects.forEach { project -> + val psiDocumentManagerBase = PsiDocumentManager.getInstance(project) as PsiDocumentManagerBase + + runInEdtAndWait { + psiDocumentManagerBase.clearUncommittedDocuments() + psiDocumentManagerBase.commitAllDocuments() + } + } + } + } + + class StatsScope(val config: StatsScopeConfig, val stats: Stats, val rootDisposable: Disposable) { + fun app(f: ApplicationScope.() -> Unit) = ApplicationScope(rootDisposable, this).use(f) + + fun measure(name: String, f: MeasurementScope.() -> Unit): List = + MeasurementScope(name, stats, config).apply(f).run() + + fun measure(name: String, f: MeasurementScope.() -> Unit, after: (() -> Unit)?): List = + MeasurementScope(name, stats, config, after = after).apply(f).run() + + fun logStatValue(name: String, value: Any) { + logMessage { "buildStatisticValue key='${stats.name}: $name' value='$value'" } + TeamCity.statValue("${stats.name}: $name", value) + } + } + + data class TypingConfig( + val fixture: Fixture, + val marker: String, + val insertString: String, + val surroundItems: String = "\n", + val typeAfterMarker: Boolean = true, + val note: String = "", + val delayMs: Long? = null + ) : AutoCloseable { + override fun close() { + fixture.close() + } + } + + class MeasurementScope( + val name: String, + val stats: Stats, + val config: StatsScopeConfig, + var before: () -> Unit = {}, + var test: (() -> T?)? = null, + var after: (() -> Unit)? = null + ) { + fun run(): List { + val t = test ?: error("test procedure isn't set") + val value = mutableListOf() + performanceTest { + name(name) + stats(stats) + warmUpIterations(config.warmup) + iterations(config.iterations) + setUp { + before() + } + test { + value.add(t.invoke()) + } + tearDown { + after?.invoke() + } + profilerEnabled(config.profile) + } + return value + } + } + + class ApplicationScope(val rootDisposable: Disposable, val stats: StatsScope) : AutoCloseable { + val application = initApp(rootDisposable) + val jdk: Sdk = initSdk(rootDisposable) + + fun project(externalProject: ExternalProject, refresh: Boolean = false, block: ProjectScope.() -> Unit) = + ProjectScope(ProjectScopeConfig(externalProject, refresh), this).use(block) + + fun project(block: ProjectWithDescriptorScope.() -> Unit) = + ProjectWithDescriptorScope(this).use(block) + + fun project(path: String, openWith: ProjectOpenAction = ProjectOpenAction.EXISTING_IDEA_PROJECT, block: ProjectScope.() -> Unit) = + ProjectScope(ProjectScopeConfig(path, openWith), this).use(block) + + fun gradleProject(path: String, refresh: Boolean = false, block: ProjectScope.() -> Unit) = + ProjectScope(ProjectScopeConfig(path, ProjectOpenAction.GRADLE_PROJECT, refresh), this).use(block) + + fun warmUpProject() = project { + descriptor { + name("helloWorld") + + kotlinFile("HelloMain") { + topFunction("main") { + param("args", "Array") + body("""println("Hello World!")""") + } + } + } + + fixture("src/HelloMain.kt").use { + highlight(it) + } + } + + override fun close() { + application?.setDataProvider(null) + } + + companion object { + fun initApp(rootDisposable: Disposable): TestApplicationManager { + val application = TestApplicationManager.getInstance() + GradleProcessOutputInterceptor.install(rootDisposable) + return application + } + + fun initSdk(rootDisposable: Disposable): Sdk { + return runWriteAction { + val jdkTableImpl = JavaAwareProjectJdkTableImpl.getInstanceEx() + val homePath = if (jdkTableImpl.internalJdk.homeDirectory!!.name == "jre") { + jdkTableImpl.internalJdk.homeDirectory!!.parent.path + } else { + jdkTableImpl.internalJdk.homePath!! + } + + val javaSdk = JavaSdk.getInstance() + val jdk = javaSdk.createJdk("1.8", homePath) + val internal = javaSdk.createJdk("IDEA jdk", homePath) + val gradle = javaSdk.createJdk(GRADLE_JDK_NAME, homePath) + + val jdkTable = getProjectJdkTableSafe() + jdkTable.addJdk(jdk, rootDisposable) + jdkTable.addJdk(internal, rootDisposable) + jdkTable.addJdk(gradle, rootDisposable) + KotlinSdkType.setUpIfNeeded() + jdk + } + } + } + } + + + class StatsScopeConfig(var name: String? = null, var warmup: Int = 2, var iterations: Int = 5, var profile: Boolean = false) + + class ProjectScopeConfig(val path: String, val openWith: ProjectOpenAction, val refresh: Boolean = false) { + val name: String = path.lastPathSegment() + + constructor(externalProject: ExternalProject, refresh: Boolean) : this(externalProject.path, externalProject.openWith, refresh) + } + + abstract class AbstractProjectScope(val app: ApplicationScope) : AutoCloseable { + abstract val project: Project + val openFiles = mutableListOf() + + fun profile(profile: ProjectProfile) { + when (profile) { + EmptyProfile -> project.disableAllInspections() + DefaultProfile -> project.initDefaultProfile() + FullProfile -> project.enableAllInspections() + is CustomProfile -> project.enableInspections(*profile.inspectionNames.toArray(emptyArray())) + } + } + + fun highlight(fixture: Fixture) = highlight(fixture.psiFile) + + fun highlight(editorFile: PsiFile?) = + editorFile?.let { + it.highlightFile() + } ?: error("editor isn't ready for highlight") + + fun moveCursor(config: TypingConfig) { + val fixture = config.fixture + val editor = fixture.editor + updateScriptDependenciesIfNeeded(fixture) + + val marker = config.marker + val tasksIdx = fixture.text.indexOf(marker) + check(tasksIdx > 0) { + "marker '$marker' not found in ${fixture.fileName}" + } + if (config.typeAfterMarker) { + editor.caretModel.moveToOffset(tasksIdx + marker.length + 1) + } else { + editor.caretModel.moveToOffset(tasksIdx - 1) + } + + for (surroundItem in config.surroundItems) { + EditorTestUtil.performTypingAction(editor, surroundItem) + } + + editor.caretModel.moveToOffset(editor.caretModel.offset - if (config.typeAfterMarker) 1 else 2) + + if (!config.typeAfterMarker) { + for (surroundItem in config.surroundItems) { + EditorTestUtil.performTypingAction(editor, surroundItem) + } + editor.caretModel.moveToOffset(editor.caretModel.offset - 2) + } + } + + fun typeAndHighlight(config: TypingConfig): List { + val string = config.insertString + for (i in string.indices) { + config.fixture.type(string[i]) + config.delayMs?.let { d -> Thread.sleep(d) } + } + return config.fixture.doHighlighting() + } + + fun updateScriptDependenciesIfNeeded(fixture: Fixture) { + val path = fixture.vFile.path + if (Fixture.isAKotlinScriptFile(path)) { + runAndMeasure("update script dependencies for $path") { + ScriptConfigurationManager.updateScriptDependenciesSynchronously(fixture.psiFile) + } + } + } + + fun enableSingleInspection(inspectionName: String) = + this.project.enableSingleInspection(inspectionName) + + fun enableAllInspections() = + this.project.enableAllInspections() + + fun editor(path: String) = + Fixture.openFileInEditor(project, path).psiFile.also { openFiles.add(it.virtualFile) } + + fun fixture(path: String): Fixture { + val fixture = Fixture.openFixture(project, path) + openFiles.add(fixture.vFile) + if (Fixture.isAKotlinScriptFile(path)) { + ScriptConfigurationManager.updateScriptDependenciesSynchronously(fixture.psiFile) + } + return fixture + } + + fun rollbackChanges() = + rollbackChanges(*openFiles.toTypedArray()) + + fun close(editorFile: PsiFile?) { + commitAllDocuments() + editorFile?.virtualFile?.let { + FileEditorManager.getInstance(project).closeFile(it) + } + } + + fun measure(vararg name: String, f: MeasurementScope.() -> Unit): List { + val after = { PsiManager.getInstance(project).dropPsiCaches() } + return app.stats.measure("${name.joinToString("-")}", f, after) + } + + fun measure(vararg name: String, fixture: Fixture, f: MeasurementScope.() -> Unit): List = + measure("${name.joinToString("-")} ${fixture.fileName}", f = f) + + override fun close() { + RunAll( + ThrowableRunnable { + project?.let { prj -> + app.application?.closeProject(prj) + } + }).run() + } + } + + class ProjectWithDescriptorScope(app: ApplicationScope) : AbstractProjectScope(app) { + private var descriptor: ProjectBuilder? = null + override val project: Project by lazy { + val builder = descriptor ?: error("project is not configured") + val openProject = builder.openProjectOperation() + + openProject.openProject().also { + openProject.postOpenProject(it) + } + } + + fun descriptor(descriptor: ProjectBuilder.() -> Unit) { + this.descriptor = ProjectBuilder().apply(descriptor) + } + } + + class ProjectScope(config: ProjectScopeConfig, app: ApplicationScope) : AbstractProjectScope(app) { + override val project: Project = initProject(config, app) + + companion object { + fun initProject(config: ProjectScopeConfig, app: ApplicationScope): Project { + val projectPath = File(config.path).canonicalPath + + UsefulTestCase.assertTrue("path ${config.path} does not exist, check README.md", File(projectPath).exists()) + + val openProject = OpenProject( + projectPath = projectPath, + projectName = config.name, + jdk = app.jdk, + projectOpenAction = config.openWith + ) + val project = ProjectOpenAction.openProject(openProject) + openProject.projectOpenAction.postOpenProject(project, openProject) + + // indexing + if (config.refresh) { + invalidateLibraryCache(project) + } + + CodeInsightTestFixtureImpl.ensureIndexesUpToDate(project) + + dispatchAllInvocationEvents() + with(DumbService.getInstance(project)) { + queueTask(UnindexedFilesUpdater(project)) + completeJustSubmittedTasks() + } + dispatchAllInvocationEvents() + + Fixture.enableAnnotatorsAndLoadDefinitions(project) + + app.application?.setDataProvider(TestDataProvider(project)) + + return project + } + } + } +} + +sealed class ProjectProfile +object EmptyProfile : ProjectProfile() +object DefaultProfile : ProjectProfile() +object FullProfile : ProjectProfile() +data class CustomProfile(val inspectionNames: List) : ProjectProfile() + +fun UsefulTestCase.suite( + suiteName: String? = null, + config: PerformanceSuite.StatsScopeConfig = PerformanceSuite.StatsScopeConfig(), + block: PerformanceSuite.StatsScope.() -> Unit +) { + PerformanceSuite.suite( + suiteName ?: this.javaClass.name, + PerformanceSuite.StatsScope(config, Stats(config.name ?: suiteName ?: name), testRootDisposable), + block + ) +} \ No newline at end of file diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/profileTools.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/profileTools.kt new file mode 100644 index 00000000000..5e079b0eaec --- /dev/null +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/profileTools.kt @@ -0,0 +1,96 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.idea.perf.util + +import com.intellij.codeInspection.ex.InspectionProfileImpl +import com.intellij.codeInspection.ex.Tools +import com.intellij.openapi.project.Project +import com.intellij.openapi.util.Disposer +import com.intellij.profile.codeInspection.InspectionProjectProfileManager +import com.intellij.profile.codeInspection.ProjectInspectionProfileManager + +class ProfileTools { + companion object { + internal fun Project.enableAllInspections() { + InspectionProfileImpl.INIT_INSPECTIONS = true + val profile = InspectionProfileImpl("all-inspections") + profile.enableAllTools(this) + replaceProfile(profile) + } + + internal fun Project.disableAllInspections() { + InspectionProfileImpl.INIT_INSPECTIONS = true + val profile = InspectionProfileImpl("no-inspections") + profile.disableAllTools(this) + replaceProfile(profile) + } + + internal fun Project.initDefaultProfile() { + val projectInspectionProfileManager = ProjectInspectionProfileManager.getInstance(this) + projectInspectionProfileManager.forceLoadSchemes() + + val projectProfile = projectInspectionProfileManager.projectProfile ?: error("project has to have non null profile name") + val profile = projectInspectionProfileManager.getProfile(projectProfile) + InspectionProfileImpl.INIT_INSPECTIONS = true + profile.initInspectionTools(this) + // disable some known `bad` inspections + profile.disableInspection(this, "SSBasedInspection") + + val enabledTools = profile.getAllEnabledInspectionTools(this) + check(enabledTools.isNotEmpty()) { + "project $name has to have at least one enabled inspection in profile" + } + InspectionProfileImpl.INIT_INSPECTIONS = false + } + + internal fun Project.enableSingleInspection(inspectionName: String) { + InspectionProfileImpl.INIT_INSPECTIONS = true + val profile = InspectionProfileImpl("$inspectionName-only") + profile.disableAllTools(this) + profile.enableTool(inspectionName, this) + + replaceProfile(profile) + } + + internal fun Project.enableInspections(vararg inspectionNames: String) { + InspectionProfileImpl.INIT_INSPECTIONS = true + val profile = InspectionProfileImpl("custom") + profile.disableAllTools(this) + inspectionNames.forEach { + profile.enableTool(it, this) + } + + replaceProfile(profile) + } + + internal fun InspectionProfileImpl.disableInspection(project: Project, vararg shortIds: String) { + shortIds.forEach { shortId -> + this.getToolsOrNull(shortId, project)?.let { it.isEnabled = false } + } + } + + private fun Project.replaceProfile(profile: InspectionProfileImpl) { + preloadProfileTools(profile, this) + val manager = InspectionProjectProfileManager.getInstance(this) as ProjectInspectionProfileManager + manager.addProfile(profile) + val prev = manager.currentProfile + manager.setCurrentProfile(profile) + Disposer.register(this, { + InspectionProfileImpl.INIT_INSPECTIONS = false + manager.setCurrentProfile(prev) + manager.deleteProfile(profile) + }) + } + + private fun preloadProfileTools( + profile: InspectionProfileImpl, + project: Project + ) { + // instantiate all tools to avoid extension loading in inconvenient moment + profile.getAllEnabledInspectionTools(project).forEach { state: Tools -> state.tool.getTool() } + } + } +} diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/testProject.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/testProject.kt new file mode 100644 index 00000000000..0d3f9f47a3e --- /dev/null +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/perf/util/testProject.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.idea.perf.util + +import org.jetbrains.kotlin.idea.testFramework.ProjectOpenAction + +/** + * @author Vladimir Ilmov + */ +class ExternalProject(val path: String, val openWith: ProjectOpenAction) { + companion object { + const val KOTLIN_PROJECT_PATH = "../perfTestProject" + + val KOTLIN_GRADLE = ExternalProject(KOTLIN_PROJECT_PATH, ProjectOpenAction.GRADLE_PROJECT) + val KOTLIN_JPS = ExternalProject(KOTLIN_PROJECT_PATH, ProjectOpenAction.EXISTING_IDEA_PROJECT) + val KOTLIN_AUTO = ExternalProject(KOTLIN_PROJECT_PATH, autoOpenAction(KOTLIN_PROJECT_PATH)) + + fun autoOpenAction(path: String): ProjectOpenAction { + return if (exists(path, ".idea", "modules.xml")) + ProjectOpenAction.EXISTING_IDEA_PROJECT.apply { println("Opening $path in iml mode.") } + else + ProjectOpenAction.GRADLE_PROJECT.apply { println("Opening $path in Gradle mode.") } + } + } +} diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/Fixture.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/Fixture.kt index 7fb8b9ea025..61bdec3139a 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/Fixture.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/Fixture.kt @@ -13,9 +13,9 @@ import com.intellij.codeInsight.daemon.impl.HighlightInfo import com.intellij.codeInsight.lookup.LookupElement import com.intellij.lang.ExternalAnnotatorsFilter import com.intellij.lang.LanguageAnnotators -import com.intellij.lang.StdLanguages import com.intellij.lang.injection.InjectedLanguageManager import com.intellij.lang.java.JavaLanguage +import com.intellij.lang.xml.XMLLanguage import com.intellij.openapi.application.runWriteAction import com.intellij.openapi.editor.Document import com.intellij.openapi.editor.Editor @@ -43,18 +43,30 @@ import junit.framework.TestCase.* import org.jetbrains.kotlin.idea.KotlinLanguage import org.jetbrains.kotlin.idea.core.script.ScriptConfigurationManager import org.jetbrains.kotlin.idea.core.script.ScriptDefinitionsManager -import org.jetbrains.kotlin.idea.core.script.settings.KotlinScriptingSettings import org.jetbrains.kotlin.idea.core.util.toPsiFile import org.jetbrains.kotlin.parsing.KotlinParserDefinition -import java.io.Closeable - -class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, val vFile: VirtualFile = psiFile.virtualFile) : Closeable { +class Fixture( + val fileName: String, + val project: Project, + val editor: Editor, + val psiFile: PsiFile, + val vFile: VirtualFile = psiFile.virtualFile +) : AutoCloseable { private var delegate = EditorTestFixture(project, editor, vFile) + private var savedText: String? = null + val document: Document get() = editor.document + val text: String + get() = document.text + + init { + storeText() + } + fun doHighlighting(): List = delegate.doHighlighting() ?: emptyList() fun type(s: String) { @@ -73,7 +85,21 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va fun complete(type: CompletionType = CompletionType.BASIC, invocationCount: Int = 1): Array = delegate.complete(type, invocationCount) ?: emptyArray() - fun revertChanges(revertChangesAtTheEnd: Boolean, text: String) { + fun storeText() { + savedText = text + } + + fun restoreText() { + savedText?.let { + try { + applyText(it) + } finally { + cleanupCaches(project) + } + } + } + + fun revertChanges(revertChangesAtTheEnd: Boolean = true, text: String) { try { if (revertChangesAtTheEnd) { // TODO: [VD] revert ? @@ -108,7 +134,10 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va dispatchAllInvocationEvents() } - override fun close() = close(project, vFile) + override fun close() { + savedText = null + close(project, vFile) + } companion object { // quite simple impl - good so far @@ -128,7 +157,7 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va InjectedLanguageManager.getInstance(project) // zillion of Dom Sem classes with(LanguageAnnotators.INSTANCE) { allForLanguage(JavaLanguage.INSTANCE) // pile of annotator classes loads - allForLanguage(StdLanguages.XML) + allForLanguage(XMLLanguage.INSTANCE) allForLanguage(KotlinLanguage.INSTANCE) } DaemonAnalyzerTestCase.assertTrue( @@ -147,7 +176,7 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va dispatchAllInvocationEvents() assertTrue(scriptDefinitionsManager.isReady()) - assertFalse(KotlinScriptingSettings.getInstance(project).isAutoReloadEnabled) + //assertFalse(KotlinScriptingSettings.getInstance(project).isAutoReloadEnabled) } @@ -157,7 +186,7 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va val editorFactory = EditorFactory.getInstance() val editor = editorFactory.getEditors(fileInEditor.document, project)[0] - return Fixture(project, editor, file) + return Fixture(fileName, project, editor, file) } private fun baseName(name: String): String { @@ -177,10 +206,10 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va val projectBaseName = baseName(project.name) val virtualFiles = FilenameIndex.getVirtualFilesByName( - project, - baseFileName, true, - GlobalSearchScope.projectScope(project) - ) + project, + baseFileName, true, + GlobalSearchScope.projectScope(project) + ) .filter { it.canonicalPath?.contains("/$projectBaseName/$name") ?: false }.toList() assertEquals( @@ -230,7 +259,7 @@ class Fixture(val project: Project, val editor: Editor, val psiFile: PsiFile, va ) { val fileInEditor = openFileInEditor(project, fileName) val editor = EditorFactory.getInstance().getEditors(fileInEditor.document, project)[0] - val fixture = Fixture(project, editor, fileInEditor.psiFile) + val fixture = Fixture(fileName, project, editor, fileInEditor.psiFile) val initialText = editor.document.text try { diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/GradleProcessOutputInterceptor.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/GradleProcessOutputInterceptor.kt index aacdb63281a..8247f9f9802 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/GradleProcessOutputInterceptor.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/GradleProcessOutputInterceptor.kt @@ -12,6 +12,7 @@ import com.intellij.openapi.externalSystem.model.task.ExternalSystemTaskNotifica import com.intellij.openapi.externalSystem.model.task.ExternalSystemTaskNotificationListener.EP_NAME as EP import com.intellij.testFramework.ExtensionTestUtil.maskExtensions import org.jetbrains.kotlin.idea.framework.GRADLE_SYSTEM_ID +import org.jetbrains.kotlin.idea.perf.util.gradleMessage import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull import java.lang.Exception import kotlin.test.assertNull @@ -44,8 +45,10 @@ private class GradleProcessOutputInterceptorImpl : GradleProcessOutputIntercepto private val buffer = StringBuilder() override fun onTaskOutput(id: ExternalSystemTaskId, text: String, stdOut: Boolean) { - if (id.projectSystemId == GRADLE_SYSTEM_ID && text.isNotEmpty()) + if (id.projectSystemId == GRADLE_SYSTEM_ID && text.isNotEmpty()) { + gradleMessage { text } buffer.append(text) + } } override fun reset() = buffer.setLength(0) diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/ProjectOpenAction.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/ProjectOpenAction.kt index c979ac58361..51acd3042cd 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/ProjectOpenAction.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/ProjectOpenAction.kt @@ -5,7 +5,6 @@ package org.jetbrains.kotlin.idea.testFramework -import com.intellij.codeInspection.ex.InspectionProfileImpl import com.intellij.ide.highlighter.ModuleFileType import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.application.runWriteAction @@ -17,18 +16,16 @@ import com.intellij.openapi.project.Project import com.intellij.openapi.project.ex.ProjectManagerEx import com.intellij.openapi.projectRoots.Sdk import com.intellij.openapi.roots.ProjectRootManager -import com.intellij.openapi.util.Disposer import com.intellij.openapi.vcs.changes.ChangeListManager import com.intellij.openapi.vcs.changes.ChangeListManagerImpl import com.intellij.openapi.vfs.LocalFileSystem import com.intellij.openapi.vfs.VirtualFileManager -import com.intellij.profile.codeInspection.ProjectInspectionProfileManager import com.intellij.testFramework.PsiTestUtil import com.intellij.testFramework.UsefulTestCase.assertTrue import com.intellij.util.io.exists import org.jetbrains.kotlin.idea.configuration.getModulesWithKotlinFiles import org.jetbrains.kotlin.idea.perf.Stats.Companion.runAndMeasure -import org.jetbrains.kotlin.idea.perf.enableAllInspectionsCompat +import org.jetbrains.kotlin.idea.perf.util.logMessage import org.jetbrains.kotlin.idea.project.getAndCacheLanguageLevelByDependencies import org.jetbrains.kotlin.idea.test.ConfigLibraryUtil import java.io.File @@ -89,11 +86,6 @@ enum class ProjectOpenAction { refreshGradleProject(projectPath, project) - assertTrue( - ModuleManager.getInstance(project).modules.isNotEmpty(), - "Gradle project $projectName at $projectPath has to have at least one module" - ) - return project } @@ -144,22 +136,6 @@ enum class ProjectOpenAction { //runWriteAction { project.save() } } - fun initDefaultProfile(project: Project) { - val projectInspectionProfileManager = ProjectInspectionProfileManager.getInstance(project) - projectInspectionProfileManager.forceLoadSchemes() - - val projectProfile = projectInspectionProfileManager.projectProfile ?: error("project has to have non null profile name") - val profile = projectInspectionProfileManager.getProfile(projectProfile) - InspectionProfileImpl.INIT_INSPECTIONS = true - profile.initInspectionTools(project) - val enabledTools = profile.getAllEnabledInspectionTools(project) - assertTrue( - "project ${project.name} has to have at least one enabled inspection in profile", - enabledTools.isNotEmpty() - ) - InspectionProfileImpl.INIT_INSPECTIONS = false - } - companion object { fun openProject(openProject: OpenProject): Project { val project = openProject.projectOpenAction.openProject( diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/compat.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/compat.kt index 042b4309d9d..4a9651c7262 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/compat.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/compat.kt @@ -12,4 +12,7 @@ import com.intellij.openapi.project.ex.ProjectManagerEx fun ProjectManagerEx.forceCloseProjectEx(project: Project, dispose: Boolean): Boolean { if (!dispose) error("dispose should be true") return this.forceCloseProject(project, true) -} \ No newline at end of file +} + +// BUNCH: 193 +typealias TestApplicationManager = com.intellij.idea.IdeaTestApplication diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/compat.kt.201 b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/compat.kt.201 index eaa3f0c6af7..e95b0761400 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/compat.kt.201 +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/compat.kt.201 @@ -12,4 +12,7 @@ import com.intellij.openapi.project.ex.ProjectManagerEx fun ProjectManagerEx.forceCloseProjectEx(project: Project, dispose: Boolean): Boolean { if (!dispose) error("dispose should be true") return this.forceCloseProject(project) -} \ No newline at end of file +} + +// BUNCH: 193 +typealias TestApplicationManager = com.intellij.testFramework.TestApplicationManager \ No newline at end of file diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/gradleRoutines.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/gradleRoutines.kt index 9bc9ff48199..d1d47e8a7e4 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/gradleRoutines.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/gradleRoutines.kt @@ -15,6 +15,7 @@ import com.intellij.openapi.project.Project import com.intellij.openapi.roots.ProjectRootManager import org.jetbrains.plugins.gradle.service.project.open.setupGradleSettings import org.jetbrains.plugins.gradle.settings.GradleProjectSettings +import org.jetbrains.plugins.gradle.settings.GradleSettings import org.jetbrains.plugins.gradle.util.GradleConstants import org.jetbrains.plugins.gradle.util.GradleLog import java.io.File @@ -26,6 +27,8 @@ fun refreshGradleProject(projectPath: String, project: Project) { dispatchAllInvocationEvents() } +const val GRADLE_JDK_NAME = "Gradle JDK" + /** * inspired by org.jetbrains.plugins.gradle.service.project.open.importProject(projectDirectory, project) */ @@ -34,7 +37,16 @@ private fun _importProject(projectPath: String, project: Project) { val projectSdk = ProjectRootManager.getInstance(project).projectSdk assertNotNull(projectSdk, "project SDK not found for ${project.name} at $projectPath") val gradleProjectSettings = GradleProjectSettings() + + GradleSettings.getInstance(project).gradleVmOptions = + "-Xmx2048m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${System.getProperty("user.dir")}" + setupGradleSettings(gradleProjectSettings, projectPath, project, projectSdk) + gradleProjectSettings.gradleJvm = GRADLE_JDK_NAME + + GradleSettings.getInstance(project).getLinkedProjectSettings(projectPath)?.let { linkedProjectSettings -> + linkedProjectSettings.gradleJvm = GRADLE_JDK_NAME + } _attachGradleProjectAndRefresh(gradleProjectSettings, project) } diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/gradleRoutines.kt.201 b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/gradleRoutines.kt.201 index 716d52cdf1b..65ef655c68d 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/gradleRoutines.kt.201 +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/gradleRoutines.kt.201 @@ -15,6 +15,7 @@ import com.intellij.openapi.project.Project import com.intellij.openapi.roots.ProjectRootManager import org.jetbrains.plugins.gradle.service.project.open.setupGradleSettings import org.jetbrains.plugins.gradle.settings.GradleProjectSettings +import org.jetbrains.plugins.gradle.settings.GradleSettings import org.jetbrains.plugins.gradle.util.GradleConstants import org.jetbrains.plugins.gradle.util.GradleLog import java.io.File @@ -26,6 +27,8 @@ fun refreshGradleProject(projectPath: String, project: Project) { dispatchAllInvocationEvents() } +const val GRADLE_JDK_NAME = "Gradle JDK" + /** * inspired by org.jetbrains.plugins.gradle.service.project.open.importProject(projectDirectory, project) */ @@ -34,7 +37,16 @@ private fun _importProject(projectPath: String, project: Project) { val projectSdk = ProjectRootManager.getInstance(project).projectSdk assertNotNull(projectSdk, "project SDK not found for ${project.name} at $projectPath") val gradleProjectSettings = GradleProjectSettings() + + GradleSettings.getInstance(project).gradleVmOptions = + "-Xmx2048m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${System.getProperty("user.dir")}" + setupGradleSettings(gradleProjectSettings, projectPath, project, projectSdk) + gradleProjectSettings.gradleJvm = GRADLE_JDK_NAME + + GradleSettings.getInstance(project).getLinkedProjectSettings(projectPath)?.let { linkedProjectSettings -> + linkedProjectSettings.gradleJvm = GRADLE_JDK_NAME + } _attachGradleProjectAndRefresh(gradleProjectSettings, project) } diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt index 406c9ad7be9..cffdcbcdec7 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt @@ -5,6 +5,9 @@ package org.jetbrains.kotlin.idea.testFramework +import com.intellij.codeInsight.daemon.DaemonCodeAnalyzer +import com.intellij.codeInsight.daemon.DaemonCodeAnalyzerSettings +import com.intellij.codeInsight.daemon.impl.DaemonCodeAnalyzerImpl import com.intellij.ide.startup.impl.StartupManagerImpl import com.intellij.lang.LanguageAnnotators import com.intellij.lang.LanguageExtensionPoint @@ -22,9 +25,8 @@ import com.intellij.testFramework.ExtensionTestUtil import com.intellij.testFramework.runInEdtAndWait import com.intellij.util.ui.UIUtil import org.jetbrains.kotlin.idea.parameterInfo.HintType +import org.jetbrains.kotlin.idea.perf.util.logMessage import org.jetbrains.kotlin.idea.test.runPostStartupActivitiesOnce -import java.io.PrintWriter -import java.io.StringWriter import java.nio.file.Paths fun commitAllDocuments() { @@ -70,10 +72,23 @@ fun dispatchAllInvocationEvents() { fun loadProjectWithName(path: String, name: String): Project? = ProjectManagerEx.getInstanceEx().loadProject(Paths.get(path), name) -fun closeProject(project: Project) { +fun TestApplicationManager.closeProject(project: Project) { + val name = project.name + val startupManagerImpl = StartupManager.getInstance(project) as StartupManagerImpl + val daemonCodeAnalyzerSettings = DaemonCodeAnalyzerSettings.getInstance() + val daemonCodeAnalyzerImpl = DaemonCodeAnalyzer.getInstance(project) as DaemonCodeAnalyzerImpl + + setDataProvider(null) + daemonCodeAnalyzerSettings.isImportHintEnabled = true // return default value to avoid unnecessary save + startupManagerImpl.checkCleared() + daemonCodeAnalyzerImpl.cleanupAfterTest() + + logMessage { "project '$name' is about to be closed" } dispatchAllInvocationEvents() val projectManagerEx = ProjectManagerEx.getInstanceEx() projectManagerEx.forceCloseProjectEx(project, true) + + logMessage { "project '$name' successfully closed" } } fun runStartupActivities(project: Project) { @@ -104,16 +119,4 @@ fun replaceWithCustomHighlighter(parentDisposable: Disposable, fromImplementatio if (filteredExtensions.size < extensions.size) { ExtensionTestUtil.maskExtensions(pointName, filteredExtensions + listOf(point), parentDisposable) } -} - -fun logMessage(message: () -> String) { - println("-- ${message()}") -} - -fun logMessage(t: Throwable, message: () -> String) { - val writer = StringWriter() - PrintWriter(writer).use { - t.printStackTrace(it) - } - println("-- ${message()}:\n$writer") } \ No newline at end of file diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt.191 b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt.191 index 0147901a3ac..0ac03c24e99 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt.191 +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt.191 @@ -25,6 +25,7 @@ import com.intellij.testFramework.PlatformTestUtil import com.intellij.testFramework.runInEdtAndWait import com.intellij.util.ui.UIUtil import org.jetbrains.kotlin.idea.parameterInfo.HintType +import org.jetbrains.kotlin.idea.perf.TestApplicationManager import org.jetbrains.kotlin.idea.test.runPostStartupActivitiesOnce import java.util.concurrent.TimeUnit import java.util.concurrent.TimeoutException @@ -74,7 +75,7 @@ fun dispatchAllInvocationEvents() { fun loadProjectWithName(path: String, name: String): Project? = ProjectManagerEx.getInstanceEx().loadProject(name, path) -fun closeProject(project: Project) { +fun TestApplicationManager.closeProject(project: Project) { dispatchAllInvocationEvents() val projectManagerEx = ProjectManagerEx.getInstanceEx() projectManagerEx.forceCloseProjectEx(project, true) @@ -133,15 +134,3 @@ fun replaceWithCustomHighlighter(parentDisposable: Disposable, fromImplementatio PlatformTestUtil.maskExtensions(pointName, filteredExtensions + listOf(point), parentDisposable) } } - -fun logMessage(message: () -> String) { - println("-- ${message()}") -} - -fun logMessage(t: Throwable, message: () -> String) { - val writer = StringWriter() - PrintWriter(writer).use { - t.printStackTrace(it) - } - println("-- ${message()}:\n$writer") -} \ No newline at end of file diff --git a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt.192 b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt.192 index c1d2f733ef8..c056dadd002 100644 --- a/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt.192 +++ b/idea/performanceTests/test/org/jetbrains/kotlin/idea/testFramework/projectRoutines.kt.192 @@ -23,6 +23,7 @@ import com.intellij.testFramework.runInEdtAndWait import com.intellij.util.ui.UIUtil import org.jetbrains.kotlin.idea.parameterInfo.HintType import org.jetbrains.kotlin.idea.test.runPostStartupActivitiesOnce +import org.jetbrains.kotlin.idea.perf.TestApplicationManager import java.io.PrintWriter import java.io.StringWriter @@ -69,7 +70,7 @@ fun dispatchAllInvocationEvents() { fun loadProjectWithName(path: String, name: String): Project? = ProjectManagerEx.getInstanceEx().loadProject(name, path) -fun closeProject(project: Project) { +fun TestApplicationManager.closeProject(project: Project) { dispatchAllInvocationEvents() val projectManagerEx = ProjectManagerEx.getInstanceEx() projectManagerEx.forceCloseProjectEx(project, true) @@ -104,15 +105,3 @@ fun replaceWithCustomHighlighter(parentDisposable: Disposable, fromImplementatio PlatformTestUtil.maskExtensions(pointName, filteredExtensions + listOf(point), parentDisposable) } } - -fun logMessage(message: () -> String) { - println("-- ${message()}") -} - -fun logMessage(t: Throwable, message: () -> String) { - val writer = StringWriter() - PrintWriter(writer).use { - t.printStackTrace(it) - } - println("-- ${message()}:\n$writer") -} \ No newline at end of file diff --git a/idea/testData/perfTest/helloKotlin/src/HelloMain.kt b/idea/testData/perfTest/helloKotlin/src/HelloMain.kt deleted file mode 100644 index 012bdca2831..00000000000 --- a/idea/testData/perfTest/helloKotlin/src/HelloMain.kt +++ /dev/null @@ -1,4 +0,0 @@ - -fun main(args: Array) { - println("Hello World!") -} diff --git a/idea/testData/perfTest/helloKotlin/src/HelloMain2.kt b/idea/testData/perfTest/helloKotlin/src/HelloMain2.kt deleted file mode 100644 index 012bdca2831..00000000000 --- a/idea/testData/perfTest/helloKotlin/src/HelloMain2.kt +++ /dev/null @@ -1,4 +0,0 @@ - -fun main(args: Array) { - println("Hello World!") -}