Disable flaky statistical benchmark tests for source sections plugin and JSR 223

This commit is contained in:
Ilya Chernikov
2017-03-22 10:19:17 +01:00
parent 3061862d31
commit 8eaeadddfa
3 changed files with 11 additions and 3 deletions
@@ -127,7 +127,9 @@ obj
}
@Test
// Note: the test is flaky, because it is statistical and the thresholds are not big enough.
// Therefore it was decided to disable it, but leave in the code in order to be able to quickly check overheads when needed.
// @Test
fun testEvalInEvalBench() {
val engine = ScriptEngineManager().getEngineByExtension("kts")!!
@@ -158,7 +158,9 @@ obj
Assert.assertEquals(7, res3)
}
@Test
// Note: the test is flaky, because it is statistical and the thresholds are not big enough.
// Therefore it was decided to disable it, but leave in the code in order to be able to quickly check overheads when needed.
// @Test
fun testEvalInEvalBench() {
val engine = ScriptEngineManager().getEngineByExtension("kts")!!
@@ -39,6 +39,7 @@ import org.jetbrains.kotlin.test.TestJdkKind
import org.jetbrains.kotlin.utils.KotlinPaths
import org.jetbrains.kotlin.utils.PathUtil
import org.jetbrains.kotlin.utils.tryConstructClassFromStringArgs
import org.junit.Ignore
import java.io.*
import java.lang.management.ManagementFactory
import java.net.URLClassLoader
@@ -126,7 +127,10 @@ class SourceSectionsTest : TestCaseWithTmpdir() {
}
}
fun testSourceSectionsRunBench() {
// Note: the test is flaky, because it is statistical and the thresholds are not big enough.
// Therefore it was decided to ignore it, but leave in the code in order to be able to quickly check overheads when needed.
@Suppress("unused")
fun ignored_testSourceSectionsRunBench() {
val mxBeans = ManagementFactory.getThreadMXBean()
val (source, _) = getTestFiles(".out").first()