Perform GC between performance test runs, add geometric mean calculation and close stats in the end

idea/idea-completion/testData/handlers/charFilter/MessageBundle1.kt

@@ -1,3 +1,5 @@
+package org.jetbrains.annotations
+
 import org.jetbrains.annotations.PropertyKey
 
 fun message(@PropertyKey(resourceBundle = "MessageBundle1.dependency") key: String) = key

idea/idea-completion/testData/handlers/charFilter/MessageBundle1.kt.after

@@ -1,3 +1,5 @@
+package org.jetbrains.annotations
+
 import org.jetbrains.annotations.PropertyKey
 
 fun message(@PropertyKey(resourceBundle = "MessageBundle1.dependency") key: String) = key

idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceAddImportTest.kt

@@ -13,7 +13,6 @@ import org.jetbrains.kotlin.idea.util.ImportInsertHelper
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.renderer.DescriptorRenderer
-import org.junit.AfterClass
 
 /**
  * inspired by @see org.jetbrains.kotlin.addImport.AbstractAddImportTest
@@ -24,11 +23,11 @@ abstract class AbstractPerformanceAddImportTest : AbstractPerformanceImportTest(
         @JvmStatic
         val stats: Stats = Stats("add-import")
 
-        @AfterClass
+        init {
-        @JvmStatic
+            // there is no @AfterClass for junit3.8
-        fun teardown() {
+            Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() }))
-            stats.close()
         }
+
     }
 
     override fun stats(): Stats = stats

idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt

@@ -18,7 +18,6 @@ import org.jetbrains.kotlin.idea.test.configureCompilerOptions
 import org.jetbrains.kotlin.idea.test.rollbackCompilerOptions
 import org.jetbrains.kotlin.test.InTextDirectivesUtils
 import org.jetbrains.kotlin.utils.addToStdlib.indexOfOrNull
-import org.junit.AfterClass
 import java.io.File
 
 /**
@@ -41,10 +40,9 @@ abstract class AbstractPerformanceCompletionHandlerTests(
         @JvmStatic
         val statsMap: MutableMap<String, Stats> = mutableMapOf()
 
-        @AfterClass
+        init {
-        @JvmStatic
+            // there is no @AfterClass for junit3.8
-        fun teardown() {
+            Runtime.getRuntime().addShutdownHook(Thread(Runnable { statsMap.values.forEach { it.close() } }))
-            statsMap.values.forEach { it.close() }
         }
     }
 

idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt

@@ -15,7 +15,6 @@ import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase
 import org.jetbrains.kotlin.idea.test.KotlinWithJdkAndRuntimeLightProjectDescriptor
 import org.jetbrains.kotlin.idea.util.application.executeWriteCommand
 import org.jetbrains.kotlin.test.InTextDirectivesUtils
-import org.junit.AfterClass
 
 /**
  * inspired by @see AbstractCompletionIncrementalResolveTest
@@ -33,10 +32,9 @@ abstract class AbstractPerformanceCompletionIncrementalResolveTest : KotlinLight
         @JvmStatic
         val stats: Stats = Stats("completion-incremental")
 
-        @AfterClass
+        init {
-        @JvmStatic
+            // there is no @AfterClass for junit3.8
-        fun teardown() {
+            Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() }))
-            stats.close()
         }
     }
 

idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt

@@ -10,7 +10,6 @@ import com.intellij.openapi.application.runWriteAction
 import com.intellij.testFramework.fixtures.impl.CodeInsightTestFixtureImpl.ensureIndexesUpToDate
 import org.jetbrains.kotlin.idea.KotlinFileType
 import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase
-import org.junit.AfterClass
 
 /**
  * inspired by @see AbstractHighlightingTest
@@ -24,10 +23,9 @@ abstract class AbstractPerformanceHighlightingTest : KotlinLightCodeInsightFixtu
         @JvmStatic
         val stats: Stats = Stats("highlight")
 
-        @AfterClass
+        init {
-        @JvmStatic
+            // there is no @AfterClass for junit3.8
-        fun teardown() {
+            Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() }))
-            stats.close()
         }
     }
 

idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt

@@ -15,7 +15,6 @@ import org.jetbrains.kotlin.idea.conversion.copy.AbstractJavaToKotlinCopyPasteCo
 import org.jetbrains.kotlin.idea.conversion.copy.ConvertJavaCopyPasteProcessor
 import org.jetbrains.kotlin.test.InTextDirectivesUtils
 import org.jetbrains.kotlin.test.KotlinTestUtils
-import org.junit.AfterClass
 import java.io.File
 
 abstract class AbstractPerformanceJavaToKotlinCopyPasteConversionTest(private val newJ2K: Boolean = false) :
@@ -27,10 +26,9 @@ abstract class AbstractPerformanceJavaToKotlinCopyPasteConversionTest(private va
 
         val stats: Array<Stats> = arrayOf(Stats("old j2k"), Stats("new j2k"))
 
-        @AfterClass
+        init {
-        @JvmStatic
+            // there is no @AfterClass for junit3.8
-        fun teardown() {
+            Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.forEach { it.close() } }))
-            stats.forEach { it.close() }
         }
     }
 

idea/performanceTests/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt

@@ -5,9 +5,6 @@
 
 package org.jetbrains.kotlin.idea.perf
 
-import org.junit.AfterClass
-import org.junit.BeforeClass
-
 class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
 
     companion object {
@@ -18,17 +15,11 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
         @JvmStatic
         val hwStats: Stats = Stats("helloWorld project")
 
-        @BeforeClass
+        init {
-        @JvmStatic
+            // there is no @AfterClass for junit3.8
-        fun setup() {
+            Runtime.getRuntime().addShutdownHook(Thread(Runnable { hwStats.close() }))
-            // things to execute once and keep around for the class
         }
 
-        @AfterClass
-        @JvmStatic
-        fun teardown() {
-            hwStats.close()
-        }
     }
 
     override fun setUp() {

idea/performanceTests/org/jetbrains/kotlin/idea/perf/Stats.kt

@@ -10,8 +10,13 @@ import java.io.*
 import kotlin.math.pow
 import kotlin.math.sqrt
 import kotlin.system.measureNanoTime
+import java.lang.ref.WeakReference
+import kotlin.math.exp
+import kotlin.math.ln
 
 class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "ValueMS", "StdDev")) : Closeable {
+    private val perfTestRawDataMs = mutableListOf<Long>()
+
     private val statsFile: File =
         File("build/stats${if (name.isNotEmpty()) "-${name.toLowerCase().replace(' ', '-')}" else ""}.csv")
             .absoluteFile
@@ -23,6 +28,22 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
         statsOutput.appendln(header.joinToString())
     }
 
+    private fun append(id: String, timingsNs: LongArray) {
+        val meanNs = timingsNs.average()
+        val meanMs = meanNs.toLong().nsToMs
+
+        val stdDivMs = (sqrt(
+            timingsNs.fold(0.0,
+                           { accumulator, next -> accumulator + (1.0 * (next - meanMs)).pow(2.0) })
+        ) / timingsNs.size).toLong().nsToMs
+
+        println("##teamcity[buildStatisticValue key='$id' value='$meanMs']")
+        println("##teamcity[buildStatisticValue key='$id stdDev' value='$stdDivMs']")
+
+        perfTestRawDataMs.addAll(timingsNs.map { it.nsToMs }.toList())
+        append(arrayOf(id, meanMs, stdDivMs))
+    }
+
     private fun append(values: Array<Any>) {
         if (values.size != header.size) {
             throw IllegalArgumentException("Expected ${header.size} values, actual ${values.size} values")
@@ -34,7 +55,8 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
     }
 
     fun append(file: String, id: String, nanoTime: Long) {
-        append(arrayOf(file, id, nanoTime.nsToMs))
+        val ms = nanoTime.nsToMs
+        append(arrayOf(file, id, ms))
     }
 
     fun <K, T> perfTest(
@@ -54,28 +76,19 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
 
             mainPhase(iterations, setUp, test, tearDown, timingsNs, namePrefix, errors)
 
-            val meanNs = timingsNs.average()
-            val meanMs = meanNs.toLong().nsToMs
-            val stdDivMs = (sqrt(
-                timingsNs.fold(0.0,
-                               { accumulator, next -> accumulator + (1.0 * (next - meanMs)).pow(2.0) })
-            ) / timingsNs.size).toLong().nsToMs
-
             for (attempt in 0 until iterations) {
-                val n = "$namePrefix #$attempt"
+                for (n in listOf("$namePrefix #$attempt", "performance test: $namePrefix #$attempt")) {
-                println("##teamcity[testStarted name='$n' captureStandardOutput='true']")
+                    println("##teamcity[testStarted name='$n' captureStandardOutput='true']")
-                if (errors[attempt] != null) {
+                    if (errors[attempt] != null) {
-                    tcPrintErrors(n, listOf(errors[attempt]!!))
+                        tcPrintErrors(n, listOf(errors[attempt]!!))
+                    }
+                    val spentMs = timingsNs[attempt].nsToMs
+                    println("##teamcity[buildStatisticValue key='$n' value='$spentMs']")
+                    println("##teamcity[testFinished name='$n' duration='$spentMs']")
                 }
-                val spentMs = timingsNs[attempt].nsToMs
-                println("##teamcity[buildStatisticValue key='$n' value='$spentMs']")
-                println("##teamcity[testFinished name='$n' duration='$spentMs']")
             }
 
-            println("##teamcity[buildStatisticValue key='$namePrefix' value='$meanMs']")
+            append(namePrefix, timingsNs)
-            println("##teamcity[buildStatisticValue key='$namePrefix stdDev' value='${stdDivMs}']")
-
-            append(arrayOf(namePrefix, meanMs, stdDivMs))
         }
     }
 
@@ -92,6 +105,8 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
         try {
             for (attempt in 0 until iterations) {
                 testData.reset()
+                triggerGC(attempt)
+
                 setUp(testData)
                 try {
                     val spentNs = measureNanoTime {
@@ -122,6 +137,9 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
         val testData = TestData<K, T>(null, null)
         for (attempt in 0 until warmUpIterations) {
             testData.reset()
+
+            triggerGC(attempt)
+
             val n = "$namePrefix warm-up #$attempt"
             println("##teamcity[testStarted name='$n' captureStandardOutput='true']")
 
@@ -156,7 +174,24 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
         }
     }
 
+    private fun triggerGC(attempt: Int) {
+        if (attempt > 0) {
+            val ref = WeakReference(IntArray(32 * 1024))
+            while (ref.get() != null) {
+                System.gc()
+                Thread.sleep(1)
+            }
+        }
+    }
+
+    private fun geomMean(data: List<Long>) = exp(data.fold(0.0, { mul, next -> mul + ln(1.0 * next) }) / data.size).toLong()
+
     override fun close() {
+        if (perfTestRawDataMs.isNotEmpty()) {
+            val geomMeanMs = geomMean(perfTestRawDataMs.toList())
+            println("##teamcity[buildStatisticValue key='$name geomMean' value='$geomMeanMs']")
+            append(arrayOf("$name geomMean", geomMeanMs, 0))
+        }
         statsOutput.flush()
         statsOutput.close()
     }