azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Perform GC between performance test runs, add geometric mean calculation and close stats in the end

Browse Source
This commit is contained in:
Vladimir Dolzhenko
2019-07-27 18:53:16 +02:00
parent 3f9480e1c5
commit 2d5a1e19d0
9 changed files with 77 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
idea/idea-completion/testData/handlers/charFilter/MessageBundle1.kt
Vendored
+2
View File
@@ -1,3 +1,5 @@
package org.jetbrains.annotations
import org.jetbrains.annotations.PropertyKey
fun message(@PropertyKey(resourceBundle = "MessageBundle1.dependency") key: String) = key
idea/idea-completion/testData/handlers/charFilter/MessageBundle1.kt.after
Vendored
+2
View File
@@ -1,3 +1,5 @@
package org.jetbrains.annotations
import org.jetbrains.annotations.PropertyKey
fun message(@PropertyKey(resourceBundle = "MessageBundle1.dependency") key: String) = key
idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceAddImportTest.kt
+4 -5
View File
@@ -13,7 +13,6 @@ import org.jetbrains.kotlin.idea.util.ImportInsertHelper
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.renderer.DescriptorRenderer
import org.junit.AfterClass
/**
* inspired by @see org.jetbrains.kotlin.addImport.AbstractAddImportTest
@@ -24,11 +23,11 @@ abstract class AbstractPerformanceAddImportTest : AbstractPerformanceImportTest(
@JvmStatic
val stats: Stats = Stats("add-import")
@AfterClass
@JvmStatic
fun teardown() {
stats.close()
init {
// there is no @AfterClass for junit3.8
Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() }))
}
}
override fun stats(): Stats = stats
idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionHandlerTests.kt
+3 -5
View File
@@ -18,7 +18,6 @@ import org.jetbrains.kotlin.idea.test.configureCompilerOptions
import org.jetbrains.kotlin.idea.test.rollbackCompilerOptions
import org.jetbrains.kotlin.test.InTextDirectivesUtils
import org.jetbrains.kotlin.utils.addToStdlib.indexOfOrNull
import org.junit.AfterClass
import java.io.File
/**
@@ -41,10 +40,9 @@ abstract class AbstractPerformanceCompletionHandlerTests(
@JvmStatic
val statsMap: MutableMap<String, Stats> = mutableMapOf()
@AfterClass
@JvmStatic
fun teardown() {
statsMap.values.forEach { it.close() }
init {
// there is no @AfterClass for junit3.8
Runtime.getRuntime().addShutdownHook(Thread(Runnable { statsMap.values.forEach { it.close() } }))
}
}
idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceCompletionIncrementalResolveTest.kt
+3 -5
View File
@@ -15,7 +15,6 @@ import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase
import org.jetbrains.kotlin.idea.test.KotlinWithJdkAndRuntimeLightProjectDescriptor
import org.jetbrains.kotlin.idea.util.application.executeWriteCommand
import org.jetbrains.kotlin.test.InTextDirectivesUtils
import org.junit.AfterClass
/**
* inspired by @see AbstractCompletionIncrementalResolveTest
@@ -33,10 +32,9 @@ abstract class AbstractPerformanceCompletionIncrementalResolveTest : KotlinLight
@JvmStatic
val stats: Stats = Stats("completion-incremental")
@AfterClass
@JvmStatic
fun teardown() {
stats.close()
init {
// there is no @AfterClass for junit3.8
Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() }))
}
}
idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceHighlightingTest.kt
+3 -5
View File
@@ -10,7 +10,6 @@ import com.intellij.openapi.application.runWriteAction
import com.intellij.testFramework.fixtures.impl.CodeInsightTestFixtureImpl.ensureIndexesUpToDate
import org.jetbrains.kotlin.idea.KotlinFileType
import org.jetbrains.kotlin.idea.test.KotlinLightCodeInsightFixtureTestCase
import org.junit.AfterClass
/**
* inspired by @see AbstractHighlightingTest
@@ -24,10 +23,9 @@ abstract class AbstractPerformanceHighlightingTest : KotlinLightCodeInsightFixtu
@JvmStatic
val stats: Stats = Stats("highlight")
@AfterClass
@JvmStatic
fun teardown() {
stats.close()
init {
// there is no @AfterClass for junit3.8
Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.close() }))
}
}
idea/performanceTests/org/jetbrains/kotlin/idea/perf/AbstractPerformanceJavaToKotlinCopyPasteConversionTest.kt
+3 -5
View File
@@ -15,7 +15,6 @@ import org.jetbrains.kotlin.idea.conversion.copy.AbstractJavaToKotlinCopyPasteCo
import org.jetbrains.kotlin.idea.conversion.copy.ConvertJavaCopyPasteProcessor
import org.jetbrains.kotlin.test.InTextDirectivesUtils
import org.jetbrains.kotlin.test.KotlinTestUtils
import org.junit.AfterClass
import java.io.File
abstract class AbstractPerformanceJavaToKotlinCopyPasteConversionTest(private val newJ2K: Boolean = false) :
@@ -27,10 +26,9 @@ abstract class AbstractPerformanceJavaToKotlinCopyPasteConversionTest(private va
val stats: Array<Stats> = arrayOf(Stats("old j2k"), Stats("new j2k"))
@AfterClass
@JvmStatic
fun teardown() {
stats.forEach { it.close() }
init {
// there is no @AfterClass for junit3.8
Runtime.getRuntime().addShutdownHook(Thread(Runnable { stats.forEach { it.close() } }))
}
}
idea/performanceTests/org/jetbrains/kotlin/idea/perf/PerformanceProjectsTest.kt
+3 -12
View File
@@ -5,9 +5,6 @@
package org.jetbrains.kotlin.idea.perf
import org.junit.AfterClass
import org.junit.BeforeClass
class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
companion object {
@@ -18,17 +15,11 @@ class PerformanceProjectsTest : AbstractPerformanceProjectsTest() {
@JvmStatic
val hwStats: Stats = Stats("helloWorld project")
@BeforeClass
@JvmStatic
fun setup() {
// things to execute once and keep around for the class
init {
// there is no @AfterClass for junit3.8
Runtime.getRuntime().addShutdownHook(Thread(Runnable { hwStats.close() }))
}
@AfterClass
@JvmStatic
fun teardown() {
hwStats.close()
}
}
override fun setUp() {
idea/performanceTests/org/jetbrains/kotlin/idea/perf/Stats.kt
+54 -19
View File
@@ -10,8 +10,13 @@ import java.io.*
import kotlin.math.pow
import kotlin.math.sqrt
import kotlin.system.measureNanoTime
import java.lang.ref.WeakReference
import kotlin.math.exp
import kotlin.math.ln
class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "ValueMS", "StdDev")) : Closeable {
private val perfTestRawDataMs = mutableListOf<Long>()
private val statsFile: File =
File("build/stats${if (name.isNotEmpty()) "-${name.toLowerCase().replace(' ', '-')}" else ""}.csv")
.absoluteFile
@@ -23,6 +28,22 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
statsOutput.appendln(header.joinToString())
}
private fun append(id: String, timingsNs: LongArray) {
val meanNs = timingsNs.average()
val meanMs = meanNs.toLong().nsToMs
val stdDivMs = (sqrt(
timingsNs.fold(0.0,
{ accumulator, next -> accumulator + (1.0 * (next - meanMs)).pow(2.0) })
) / timingsNs.size).toLong().nsToMs
println("##teamcity[buildStatisticValue key='$id' value='$meanMs']")
println("##teamcity[buildStatisticValue key='$id stdDev' value='$stdDivMs']")
perfTestRawDataMs.addAll(timingsNs.map { it.nsToMs }.toList())
append(arrayOf(id, meanMs, stdDivMs))
}
private fun append(values: Array<Any>) {
if (values.size != header.size) {
throw IllegalArgumentException("Expected ${header.size} values, actual ${values.size} values")
@@ -34,7 +55,8 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
}
fun append(file: String, id: String, nanoTime: Long) {
append(arrayOf(file, id, nanoTime.nsToMs))
val ms = nanoTime.nsToMs
append(arrayOf(file, id, ms))
}
fun <K, T> perfTest(
@@ -54,28 +76,19 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
mainPhase(iterations, setUp, test, tearDown, timingsNs, namePrefix, errors)
val meanNs = timingsNs.average()
val meanMs = meanNs.toLong().nsToMs
val stdDivMs = (sqrt(
timingsNs.fold(0.0,
{ accumulator, next -> accumulator + (1.0 * (next - meanMs)).pow(2.0) })
) / timingsNs.size).toLong().nsToMs
for (attempt in 0 until iterations) {
val n = "$namePrefix #$attempt"
println("##teamcity[testStarted name='$n' captureStandardOutput='true']")
if (errors[attempt] != null) {
tcPrintErrors(n, listOf(errors[attempt]!!))
for (n in listOf("$namePrefix #$attempt", "performance test: $namePrefix #$attempt")) {
println("##teamcity[testStarted name='$n' captureStandardOutput='true']")
if (errors[attempt] != null) {
tcPrintErrors(n, listOf(errors[attempt]!!))
}
val spentMs = timingsNs[attempt].nsToMs
println("##teamcity[buildStatisticValue key='$n' value='$spentMs']")
println("##teamcity[testFinished name='$n' duration='$spentMs']")
}
val spentMs = timingsNs[attempt].nsToMs
println("##teamcity[buildStatisticValue key='$n' value='$spentMs']")
println("##teamcity[testFinished name='$n' duration='$spentMs']")
}
println("##teamcity[buildStatisticValue key='$namePrefix' value='$meanMs']")
println("##teamcity[buildStatisticValue key='$namePrefix stdDev' value='${stdDivMs}']")
append(arrayOf(namePrefix, meanMs, stdDivMs))
append(namePrefix, timingsNs)
}
}
@@ -92,6 +105,8 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
try {
for (attempt in 0 until iterations) {
testData.reset()
triggerGC(attempt)
setUp(testData)
try {
val spentNs = measureNanoTime {
@@ -122,6 +137,9 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
val testData = TestData<K, T>(null, null)
for (attempt in 0 until warmUpIterations) {
testData.reset()
triggerGC(attempt)
val n = "$namePrefix warm-up #$attempt"
println("##teamcity[testStarted name='$n' captureStandardOutput='true']")
@@ -156,7 +174,24 @@ class Stats(val name: String = "", val header: Array<String> = arrayOf("Name", "
}
}
private fun triggerGC(attempt: Int) {
if (attempt > 0) {
val ref = WeakReference(IntArray(32 * 1024))
while (ref.get() != null) {
System.gc()
Thread.sleep(1)
}
}
}
private fun geomMean(data: List<Long>) = exp(data.fold(0.0, { mul, next -> mul + ln(1.0 * next) }) / data.size).toLong()
override fun close() {
if (perfTestRawDataMs.isNotEmpty()) {
val geomMeanMs = geomMean(perfTestRawDataMs.toList())
println("##teamcity[buildStatisticValue key='$name geomMean' value='$geomMeanMs']")
append(arrayOf("$name geomMean", geomMeanMs, 0))
}
statsOutput.flush()
statsOutput.close()
}