From 2d5a1e19d047e1a6ced81f3a460390083e073eb8 Mon Sep 17 00:00:00 2001 From: Vladimir Dolzhenko Date: Sat, 27 Jul 2019 18:53:16 +0200 Subject: [PATCH] Perform GC between performance test runs, add geometric mean calculation and close stats in the end --- .../handlers/charFilter/MessageBundle1.kt | 2 + .../charFilter/MessageBundle1.kt.after | 2 + .../perf/AbstractPerformanceAddImportTest.kt | 9 +-- ...stractPerformanceCompletionHandlerTests.kt | 8 +- ...ormanceCompletionIncrementalResolveTest.kt | 8 +- .../AbstractPerformanceHighlightingTest.kt | 8 +- ...anceJavaToKotlinCopyPasteConversionTest.kt | 8 +- .../idea/perf/PerformanceProjectsTest.kt | 15 +--- .../org/jetbrains/kotlin/idea/perf/Stats.kt | 73 ++++++++++++++----- 9 files changed, 77 insertions(+), 56 deletions(-) diff --git a/idea/idea-completion/testData/handlers/charFilter/MessageBundle1.kt b/idea/idea-completion/testData/handlers/charFilter/MessageBundle1.kt index c1544247cc5..be978a4da94 100644 --- a/idea/idea-completion/testData/handlers/charFilter/MessageBundle1.kt +++ b/idea/idea-completion/testData/handlers/charFilter/MessageBundle1.kt @@ -1,3 +1,5 @@ +package org.jetbrains.annotations + import org.jetbrains.annotations.PropertyKey fun message(@PropertyKey(resourceBundle = "MessageBundle1.dependency") key: String) = key diff --git a/idea/idea-completion/testData/handlers/charFilter/MessageBundle1.kt.after b/idea/idea-completion/testData/handlers/charFilter/MessageBundle1.kt.after index e34be94fa14..ec56a63c2a1 100644 --- a/idea/idea-completion/testData/handlers/charFilter/MessageBundle1.kt.after +++ b/idea/idea-completion/testData/handlers/charFilter/MessageBundle1.kt.after @@ -1,3 +1,5 @@ +package org.jetbrains.annotations + import org.jetbrains.annotations.PropertyKey fun message(@PropertyKey(resourceBundle = "MessageBundle1.dependency") key: String) = key diff --git a/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceAddImportTest.kt b/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceAddImportTest.kt index fe7c84aa97b..aca8224d51d 100644 --- a/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceAddImportTest.kt +++ b/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceAddImportTest.kt @@ -13,7 +13,6 @@ import org.jetbrains.kotlin.idea.util.ImportInsertHelper import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.renderer.DescriptorRenderer -import org.junit.AfterClass /** * inspired by @see org.jetbrains.kotlin.addImport.AbstractAddImportTest @@ -24,11 +23,11 @@ abstract class AbstractPerformanceAddImportTest : AbstractPerformanceImportTest( @JvmStatic val stats: Stats = Stats("add-import") - @AfterClass - @JvmStatic - fun teardown() { - stats.close() + init { + // there is no @AfterClass for junit3.8 + Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() })) } + } override fun stats(): Stats = stats diff --git a/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt b/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt index ea0d1fd2b19..1e297977937 100644 --- a/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt +++ b/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt @@ -18,7 +18,6 @@ import org.jetbrains.kotlin.idea.test.configureCompilerOptions import org.jetbrains.kotlin.idea.test.rollbackCompilerOptions import org.jetbrains.kotlin.test.InTextDirectivesUtils import org.jetbrains.kotlin.utils.addToStdlib.indexOfOrNull -import org.junit.AfterClass import java.io.File /** @@ -41,10 +40,9 @@ abstract class AbstractPerformanceCompletionHandlerTests( @JvmStatic val statsMap: MutableMap = mutableMapOf() - @AfterClass - @JvmStatic - fun teardown() { - statsMap.values.forEach { it.close() } + init { + // there is no @AfterClass for junit3.8 + Runtime.getRuntime().addShutdownHook(Thread(Runnable { statsMap.values.forEach { it.close() } })) } } diff --git a/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt b/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt index f2f90588367..5a7db5065b1 100644 --- a/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt +++ b/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt @@ -15,7 +15,6 @@ import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase import org.jetbrains.kotlin.idea.test.KotlinWithJdkAndRuntimeLightProjectDescriptor import org.jetbrains.kotlin.idea.util.application.executeWriteCommand import org.jetbrains.kotlin.test.InTextDirectivesUtils -import org.junit.AfterClass /** * inspired by @see AbstractCompletionIncrementalResolveTest @@ -33,10 +32,9 @@ abstract class AbstractPerformanceCompletionIncrementalResolveTest : KotlinLight @JvmStatic val stats: Stats = Stats("completion-incremental") - @AfterClass - @JvmStatic - fun teardown() { - stats.close() + init { + // there is no @AfterClass for junit3.8 + Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() })) } } diff --git a/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt b/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt index 9bac9fa6ff5..a189e9c4c11 100644 --- a/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt +++ b/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt @@ -10,7 +10,6 @@ import com.intellij.openapi.application.runWriteAction import com.intellij.testFramework.fixtures.impl.CodeInsightTestFixtureImpl.ensureIndexesUpToDate import org.jetbrains.kotlin.idea.KotlinFileType import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase -import org.junit.AfterClass /** * inspired by @see AbstractHighlightingTest @@ -24,10 +23,9 @@ abstract class AbstractPerformanceHighlightingTest : KotlinLightCodeInsightFixtu @JvmStatic val stats: Stats = Stats("highlight") - @AfterClass - @JvmStatic - fun teardown() { - stats.close() + init { + // there is no @AfterClass for junit3.8 + Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() })) } } diff --git a/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt b/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt index 3cfc238f660..ca65a11423e 100644 --- a/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt +++ b/idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt @@ -15,7 +15,6 @@ import org.jetbrains.kotlin.idea.conversion.copy.AbstractJavaToKotlinCopyPasteCo import org.jetbrains.kotlin.idea.conversion.copy.ConvertJavaCopyPasteProcessor import org.jetbrains.kotlin.test.InTextDirectivesUtils import org.jetbrains.kotlin.test.KotlinTestUtils -import org.junit.AfterClass import java.io.File abstract class AbstractPerformanceJavaToKotlinCopyPasteConversionTest(private val newJ2K: Boolean = false) : @@ -27,10 +26,9 @@ abstract class AbstractPerformanceJavaToKotlinCopyPasteConversionTest(private va val stats: Array = arrayOf(Stats("old j2k"), Stats("new j2k")) - @AfterClass - @JvmStatic - fun teardown() { - stats.forEach { it.close() } + init { + // there is no @AfterClass for junit3.8 + Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.forEach { it.close() } })) } } diff --git a/idea/performanceTests/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt b/idea/performanceTests/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt index 16927021606..682c485a05b 100644 --- a/idea/performanceTests/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt +++ b/idea/performanceTests/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt @@ -5,9 +5,6 @@ package org.jetbrains.kotlin.idea.perf -import org.junit.AfterClass -import org.junit.BeforeClass - class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { companion object { @@ -18,17 +15,11 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() { @JvmStatic val hwStats: Stats = Stats("helloWorld project") - @BeforeClass - @JvmStatic - fun setup() { - // things to execute once and keep around for the class + init { + // there is no @AfterClass for junit3.8 + Runtime.getRuntime().addShutdownHook(Thread(Runnable { hwStats.close() })) } - @AfterClass - @JvmStatic - fun teardown() { - hwStats.close() - } } override fun setUp() { diff --git a/idea/performanceTests/org/jetbrains/kotlin/idea/perf/Stats.kt b/idea/performanceTests/org/jetbrains/kotlin/idea/perf/Stats.kt index 58889908b7f..422640bae49 100644 --- a/idea/performanceTests/org/jetbrains/kotlin/idea/perf/Stats.kt +++ b/idea/performanceTests/org/jetbrains/kotlin/idea/perf/Stats.kt @@ -10,8 +10,13 @@ import java.io.* import kotlin.math.pow import kotlin.math.sqrt import kotlin.system.measureNanoTime +import java.lang.ref.WeakReference +import kotlin.math.exp +import kotlin.math.ln class Stats(val name: String = "", val header: Array = arrayOf("Name", "ValueMS", "StdDev")) : Closeable { + private val perfTestRawDataMs = mutableListOf() + private val statsFile: File = File("build/stats${if (name.isNotEmpty()) "-${name.toLowerCase().replace(' ', '-')}" else ""}.csv") .absoluteFile @@ -23,6 +28,22 @@ class Stats(val name: String = "", val header: Array = arrayOf("Name", " statsOutput.appendln(header.joinToString()) } + private fun append(id: String, timingsNs: LongArray) { + val meanNs = timingsNs.average() + val meanMs = meanNs.toLong().nsToMs + + val stdDivMs = (sqrt( + timingsNs.fold(0.0, + { accumulator, next -> accumulator + (1.0 * (next - meanMs)).pow(2.0) }) + ) / timingsNs.size).toLong().nsToMs + + println("##teamcity[buildStatisticValue key='$id' value='$meanMs']") + println("##teamcity[buildStatisticValue key='$id stdDev' value='$stdDivMs']") + + perfTestRawDataMs.addAll(timingsNs.map { it.nsToMs }.toList()) + append(arrayOf(id, meanMs, stdDivMs)) + } + private fun append(values: Array) { if (values.size != header.size) { throw IllegalArgumentException("Expected ${header.size} values, actual ${values.size} values") @@ -34,7 +55,8 @@ class Stats(val name: String = "", val header: Array = arrayOf("Name", " } fun append(file: String, id: String, nanoTime: Long) { - append(arrayOf(file, id, nanoTime.nsToMs)) + val ms = nanoTime.nsToMs + append(arrayOf(file, id, ms)) } fun perfTest( @@ -54,28 +76,19 @@ class Stats(val name: String = "", val header: Array = arrayOf("Name", " mainPhase(iterations, setUp, test, tearDown, timingsNs, namePrefix, errors) - val meanNs = timingsNs.average() - val meanMs = meanNs.toLong().nsToMs - val stdDivMs = (sqrt( - timingsNs.fold(0.0, - { accumulator, next -> accumulator + (1.0 * (next - meanMs)).pow(2.0) }) - ) / timingsNs.size).toLong().nsToMs - for (attempt in 0 until iterations) { - val n = "$namePrefix #$attempt" - println("##teamcity[testStarted name='$n' captureStandardOutput='true']") - if (errors[attempt] != null) { - tcPrintErrors(n, listOf(errors[attempt]!!)) + for (n in listOf("$namePrefix #$attempt", "performance test: $namePrefix #$attempt")) { + println("##teamcity[testStarted name='$n' captureStandardOutput='true']") + if (errors[attempt] != null) { + tcPrintErrors(n, listOf(errors[attempt]!!)) + } + val spentMs = timingsNs[attempt].nsToMs + println("##teamcity[buildStatisticValue key='$n' value='$spentMs']") + println("##teamcity[testFinished name='$n' duration='$spentMs']") } - val spentMs = timingsNs[attempt].nsToMs - println("##teamcity[buildStatisticValue key='$n' value='$spentMs']") - println("##teamcity[testFinished name='$n' duration='$spentMs']") } - println("##teamcity[buildStatisticValue key='$namePrefix' value='$meanMs']") - println("##teamcity[buildStatisticValue key='$namePrefix stdDev' value='${stdDivMs}']") - - append(arrayOf(namePrefix, meanMs, stdDivMs)) + append(namePrefix, timingsNs) } } @@ -92,6 +105,8 @@ class Stats(val name: String = "", val header: Array = arrayOf("Name", " try { for (attempt in 0 until iterations) { testData.reset() + triggerGC(attempt) + setUp(testData) try { val spentNs = measureNanoTime { @@ -122,6 +137,9 @@ class Stats(val name: String = "", val header: Array = arrayOf("Name", " val testData = TestData(null, null) for (attempt in 0 until warmUpIterations) { testData.reset() + + triggerGC(attempt) + val n = "$namePrefix warm-up #$attempt" println("##teamcity[testStarted name='$n' captureStandardOutput='true']") @@ -156,7 +174,24 @@ class Stats(val name: String = "", val header: Array = arrayOf("Name", " } } + private fun triggerGC(attempt: Int) { + if (attempt > 0) { + val ref = WeakReference(IntArray(32 * 1024)) + while (ref.get() != null) { + System.gc() + Thread.sleep(1) + } + } + } + + private fun geomMean(data: List) = exp(data.fold(0.0, { mul, next -> mul + ln(1.0 * next) }) / data.size).toLong() + override fun close() { + if (perfTestRawDataMs.isNotEmpty()) { + val geomMeanMs = geomMean(perfTestRawDataMs.toList()) + println("##teamcity[buildStatisticValue key='$name geomMean' value='$geomMeanMs']") + append(arrayOf("$name geomMean", geomMeanMs, 0)) + } statsOutput.flush() statsOutput.close() }