diff --git a/generators/build.gradle.kts b/generators/build.gradle.kts index f33ffcd0362..7492476f3fa 100644 --- a/generators/build.gradle.kts +++ b/generators/build.gradle.kts @@ -41,6 +41,7 @@ dependencies { testCompile(projectTests(":nj2k")) testCompile(projectTests(":libraries:tools:new-project-wizard:new-project-wizard-cli")) testCompile(projectTests(":idea:idea-android")) + testCompile(projectTests(":idea:performanceTests")) testCompile(projectTests(":idea:scripting-support")) testCompile(projectTests(":jps-plugin")) testCompile(projectTests(":plugins:jvm-abi-gen")) diff --git a/idea/performanceTests/README.md b/idea/performanceTests/README.md index 22172ed4f6f..0b6a8511994 100644 --- a/idea/performanceTests/README.md +++ b/idea/performanceTests/README.md @@ -1,42 +1,70 @@ # Goal -The main goal of these performance tests is to collect statistics over +The main goal of Kotlin IDE plugin performance tests is to collect statistics over routines (open project, highlighting, inspection, autocompletion etc) for further analysis like anomalies, degradations and reference for optimizations. ## Resources -- Clone another copy of kotlin project to `${KOTLIN_PROJECT}/../perfTestProject` -- `${KOTLIN_PROJECT}/idea/testData/perfTest/helloKotlin` as _hello world_ project in Kotlin -## Performance test types +You need an extra copy of kotlin project located at `${KOTLIN_PROJECT}/../perfTestProject` -### AbstractKotlinProjectsPerformanceTest +## Run -Subclasses of `AbstractKotlinProjectsPerformanceTest` provide statistics -(execution time in **ms**) over some **particular** (semi-synthetic) actions. +Run all Kotlin IDE plugin performance tests with -The output is provided to console in TeamCity format like +`$ gradle idea-plugin-performance-tests` + +## Performance test + +Output is provided to console in TeamCity format like ``` -##teamcity[testSuiteStarted name='Kotlin'] -##teamcity[testStarted name='Project opening perfTestProject' captureStandardOutput='true'] -##teamcity[testMetadata key='Project opening perfTestProject' type='number' value='63'] -##teamcity[testFinished name='Project opening perfTestProject' duration='63'] -##teamcity[testSuiteFinished name='Kotlin'] +##teamcity[testSuiteStarted name='Field'] +##teamcity[testStarted name='highlight: Field warm-up #0' captureStandardOutput='true'] +##teamcity[buildStatisticValue key='highlight: Field warm-up #0' value='142'] +##teamcity[testFinished name='highlight: Field warm-up #0' duration='142'] +.... +##teamcity[testStarted name='highlight: Field #0' captureStandardOutput='true'] +##teamcity[buildStatisticValue key='highlight: Field #0 counter "Call resolve": time' value='8'] +##teamcity[buildStatisticValue key='highlight: Field #0' value='67'] +##teamcity[testFinished name='highlight: Field #0' duration='67'] +.... +##teamcity[testStarted name='highlight: Field #19' captureStandardOutput='true'] +##teamcity[buildStatisticValue key='highlight: Field #19' value='56'] +##teamcity[testFinished name='highlight: Field #19' duration='56'] + +##teamcity[testStarted name='highlight: Field : mean' captureStandardOutput='true'] +##teamcity[buildStatisticValue key='highlight: Field' value='57'] +##teamcity[testFinished name='highlight: Field : mean' duration='57'] + +##teamcity[testStarted name='highlight: Field : stdDev' captureStandardOutput='true'] +##teamcity[buildStatisticValue key='highlight: Field stdDev' value='0'] +##teamcity[testFinished name='highlight: Field : stdDev' duration='0'] + +##teamcity[testStarted name='highlight: Field : geomMean' captureStandardOutput='true'] +##teamcity[buildStatisticValue key='highlight: Field geomMean' value='57'] +##teamcity[testFinished name='highlight: Field : geomMean' duration='57'] + +##teamcity[testStarted name='highlight: Field stability' captureStandardOutput='true'] +##teamcity[buildStatisticValue key='highlight: Field stability' value='2'] +##teamcity[testFinished name='highlight: Field stability'] + +##teamcity[testSuiteFinished name='Field'] ``` -as well in `build/stats.csv` +as well in CVS files at `build/stats*.csv`, e.g. `build/stats-highlight.csv`: - File | ProcessID | Time + Name | ValueMS | StdDev --- | --- | --- - HelloMain.kt | Highlighting file warm-up HelloMain.kt | 114 - KtFile.kt | Highlighting file KtFile.kt | 1255 + highlight: NonNullAssertion | 56.1 | 1.86 + highlight: PropertiesWithPropertyDeclarations | 308.1 | 11.05 + highlight: NamedArguments | 67.15 | 1.23 + highlight: Annotations | 84.85 | 1.18 + highlight: VariablesAsFunctions | 125.5 | 2.11 + highlight: Enums | 84.4 | 1.31 + highlight: Field | 67.8 | 1.74 + -### WholeProjectPerformanceTest +## PerfTest DSL -**TBD**: drop `@Ignore` from some test cases - -Subclasses of `WholeProjectPerformanceTest` collect performance metrics to -figure out some significant anomalies (e.g. inspection takes ages or -even **fails**) over huge number of files (like `kotlin` project itself) -on various **real** use cases. +TBD \ No newline at end of file diff --git a/idea/performanceTests/build.gradle.kts b/idea/performanceTests/build.gradle.kts index dff5e0b3e62..b0ab24084a3 100644 --- a/idea/performanceTests/build.gradle.kts +++ b/idea/performanceTests/build.gradle.kts @@ -76,3 +76,5 @@ projectTest(taskName = "performanceTest") { } } } + +testsJar() \ No newline at end of file