From 53b31a20ca8f94946d66c33fde48f802ab2844c0 Mon Sep 17 00:00:00 2001 From: Ilya Muradyan Date: Thu, 4 Jun 2020 13:43:58 +0300 Subject: [PATCH] Refactor REPL IDE services testing configuration and add new tests --- .../scripting/ide_services/JvmReplTest.kt | 8 +- .../ReplCompletionAndErrorsAnalysisTest.kt | 444 +++++------------- .../test_util/testConfiguration.kt | 293 ++++++++++++ 3 files changed, 412 insertions(+), 333 deletions(-) create mode 100644 plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/test_util/testConfiguration.kt diff --git a/plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/JvmReplTest.kt b/plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/JvmReplTest.kt index 230f0a7d2c6..3c2597c04c5 100644 --- a/plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/JvmReplTest.kt +++ b/plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/JvmReplTest.kt @@ -270,9 +270,7 @@ class JvmIdeServicesTest : TestCase() { } } -// Artificial split into several testsuites, to speed up parallel testing -class LegacyReplTestLong1 : TestCase() { - +class LegacyReplTestLong : TestCase() { fun test256Evals() { JvmTestRepl() .use { repl -> @@ -302,10 +300,6 @@ class LegacyReplTestLong1 : TestCase() { assertEquals(evaluated.toString(), evals, (evaluated?.result as ResultValue.Value?)?.value) } } -} - -// Artificial split into several testsuites, to speed up parallel testing -class LegacyReplTestLong2 : TestCase() { fun testReplSlowdownKt22740() { JvmTestRepl() diff --git a/plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/ReplCompletionAndErrorsAnalysisTest.kt b/plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/ReplCompletionAndErrorsAnalysisTest.kt index abff6ae8ce6..591d89579ec 100644 --- a/plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/ReplCompletionAndErrorsAnalysisTest.kt +++ b/plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/ReplCompletionAndErrorsAnalysisTest.kt @@ -6,23 +6,21 @@ package org.jetbrains.kotlin.scripting.ide_services import junit.framework.TestCase -import kotlinx.coroutines.runBlocking -import org.jetbrains.kotlin.scripting.ide_services.compiler.KJvmReplCompilerWithIdeServices import org.jetbrains.kotlin.scripting.ide_services.compiler.completion import org.jetbrains.kotlin.scripting.ide_services.compiler.filterOutShadowedDescriptors import org.jetbrains.kotlin.scripting.ide_services.compiler.nameFilter -import org.jetbrains.kotlin.scripting.ide_services.test_util.SourceCodeTestImpl -import org.jetbrains.kotlin.scripting.ide_services.test_util.simpleScriptCompilationConfiguration -import org.jetbrains.kotlin.scripting.ide_services.test_util.toList -import org.junit.Assert +import org.jetbrains.kotlin.scripting.ide_services.test_util.* +import org.jetbrains.kotlin.scripting.resolve.skipExtensionsResolutionForImplicits +import org.jetbrains.kotlin.scripting.resolve.skipExtensionsResolutionForImplicitsExceptInnermost import org.junit.Ignore import org.junit.Test import java.io.Writer -import java.util.concurrent.atomic.AtomicInteger -import kotlin.reflect.KProperty -import kotlin.reflect.KProperty0 import kotlin.script.experimental.api.* -import kotlin.system.measureTimeMillis +import kotlin.script.experimental.jvm.jvm +import kotlin.script.experimental.jvm.updateClasspath +import kotlin.script.experimental.jvm.util.classpathFromClass + +typealias TestRunConfigurator = (TestConf.Run).() -> Unit class ReplCompletionAndErrorsAnalysisTest : TestCase() { @Test @@ -293,43 +291,6 @@ class ReplCompletionAndErrorsAnalysisTest : TestCase() { } } - @Test - fun testLongRunningCompletion() = test { - // This test normally completes in about half a minute - // Log should show slow _linear_ compilation/completion time growth - - val compileWriter = System.out.writer() // FileWriter("$csvDir/compilations.csv") - val completeWriter = System.out.writer() // FileWriter("$csvDir/completions.csv") - - for (i in 1..230) { - run { - code = """ - val dataFrame = mapOf("x" to 45) - val e = "str" - """.trimIndent() - doCompile - - loggingInfo = CSVLoggingInfo(compile = CSVLoggingInfoItem(compileWriter, i, "compile;")) - } - - run { - code = """ - val x = mapOf("a" to dataFrame., "b" to 12, e to 42) - """.trimIndent() - cursor = 31 - - doComplete - expect { - completions.mode = ComparisonType.INCLUDES - addCompletion("entries", "entries", "Set>", "property") - } - - loggingInfo = CSVLoggingInfo(complete = CSVLoggingInfoItem(completeWriter, i, "complete;")) - } - - } - } - @Test fun testImplicitExtensions() = test { run { @@ -371,7 +332,43 @@ class ReplCompletionAndErrorsAnalysisTest : TestCase() { run(setupDefaultImportsCompletionRun) } - private val setupDefaultImportsCompletionRun: (TestConf.Run).() -> Unit = { + @Test + fun testLongRunningCompilationWithReceiver() = test { + // This test normally completes in about 8-13s + // Removing skip* configuration parameters should slow down the test (2-3 times) + + val conf = ScriptCompilationConfiguration { + jvm { + updateClasspath(classpathFromClass()) + } + implicitReceivers(TestReceiver1::class, TestReceiver2::class) + skipExtensionsResolutionForImplicits(KotlinType(TestReceiver1::class)) + skipExtensionsResolutionForImplicitsExceptInnermost(KotlinType(TestReceiver2::class)) + } + + val writer = System.out.writer() + for (i in 1..200) { + run(longCompilationRun(writer, i, conf)) + run { + compilationConfiguration = conf + + code = """ + val x = xyz + """.trimIndent() + cursor = 11 + + expect { + completions.mode = ComparisonType.EQUALS + addCompletion("xyz1", "xyz1", "Int", "property") + addCompletion("xyz2", "xyz2", "Int", "property") + } + + loggingInfo = CSVLoggingInfo(complete = CSVLoggingInfoItem(writer, i, "complete;")) + } + } + } + + private val setupDefaultImportsCompletionRun: TestRunConfigurator = { compilationConfiguration = ScriptCompilationConfiguration { defaultImports(listOf("kotlin.math.atan")) } @@ -384,283 +381,78 @@ class ReplCompletionAndErrorsAnalysisTest : TestCase() { addCompletion("atan(", "atan(Double)", "Double", "method") } } - - class TestConf { - private val runs = mutableListOf() - - fun run(setup: (Run).() -> Unit) { - val r = Run() - r.setup() - runs.add(r) - } - - fun collect() = runs.map { it.collect() } - - class Run { - private var _doCompile = false - val doCompile: Unit - get() { - _doCompile = true - } - - private var _doComplete = false - val doComplete: Unit - get() { - _doComplete = true - } - - private var _doErrorCheck = false - val doErrorCheck: Unit - get() { - _doErrorCheck = true - } - - var cursor: Int? = null - var compilationConfiguration: ScriptCompilationConfiguration? = null - var code: String = "" - private var _expected: Expected = Expected(this) - - var loggingInfo: CSVLoggingInfo? = null - - fun expect(setup: (Expected).() -> Unit) { - _expected = Expected(this) - _expected.setup() - } - - fun collect(): Pair { - return RunRequest( - cursor, - code, - _doCompile, - _doComplete, - _doErrorCheck, - compilationConfiguration, - loggingInfo - ) to _expected.collect() - } - - class Expected(private val run: Run) { - val completions = ExpectedList(run::doComplete) - fun addCompletion(text: String, displayText: String, tail: String, icon: String) { - completions.add(SourceCodeCompletionVariant(text, displayText, tail, icon)) - } - - val errors = ExpectedList(run::doErrorCheck) - fun addError(startLine: Int, startCol: Int, endLine: Int, endCol: Int, message: String, severity: String) { - errors.add( - ScriptDiagnostic( - ScriptDiagnostic.unspecifiedError, - message, - ScriptDiagnostic.Severity.valueOf(severity), - location = SourceCode.Location( - SourceCode.Position(startLine, startCol), - SourceCode.Position(endLine, endCol) - ) - ) - ) - } - - var resultType: String? by ExpectedNullableVar(run::doErrorCheck) - - fun collect(): ExpectedResult { - return ExpectedResult(completions, errors, resultType) - } - } - - } - } - - private fun test(setup: (TestConf).() -> Unit) { - val test = TestConf() - test.setup() - runBlocking { checkEvaluateInRepl(simpleScriptCompilationConfiguration, test.collect()) } - } - - enum class ComparisonType { - COMPARE_SIZE, INCLUDES, EQUALS, DONT_CHECK - } - - data class CSVLoggingInfoItem( - val writer: Writer, - val xValue: Int, - val prefix: String = "", - ) { - fun writeValue(value: Any) { - writer.write("$prefix$xValue;$value\n") - writer.flush() - } - } - - data class CSVLoggingInfo( - val compile: CSVLoggingInfoItem? = null, - val complete: CSVLoggingInfoItem? = null, - val analyze: CSVLoggingInfoItem? = null, - ) - - data class RunRequest( - val cursor: Int?, - val code: String, - val doCompile: Boolean, - val doComplete: Boolean, - val doErrorCheck: Boolean, - val compilationConfiguration: ScriptCompilationConfiguration?, - val loggingInfo: CSVLoggingInfo?, - ) - - interface ExpectedOptions { - val mode: ComparisonType - val size: Int - } - - class ExpectedList(private val runProperty: KProperty0) : ExpectedOptions { - val list = mutableListOf() - - override var mode = ComparisonType.DONT_CHECK - override var size = 0 - set(value) { - if (mode == ComparisonType.DONT_CHECK) - mode = ComparisonType.COMPARE_SIZE - runProperty.get() - field = value - } - - fun add(elem: T) { - if (mode == ComparisonType.DONT_CHECK) - mode = ComparisonType.EQUALS - runProperty.get() - list.add(elem) - } - } - - class ExpectedNullableVar(private val runProperty: KProperty0) { - private var variable: T? = null - - operator fun getValue(thisRef: Any?, property: KProperty<*>): T? = variable - operator fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) { - runProperty.get() - variable = value - } - } - - data class ExpectedResult( - val completions: ExpectedList, - val errors: ExpectedList, - val resultType: String?, - ) - - data class ActualResult( - val completions: List, - val errors: List, - val resultType: String?, - ) - - private val currentLineCounter = AtomicInteger() - - private fun nextCodeLine(code: String): SourceCode = - SourceCodeTestImpl( - currentLineCounter.getAndIncrement(), - code - ) - - private suspend fun evaluateInRepl( - compilationConfiguration: ScriptCompilationConfiguration, - snippets: List - ): List> { - val compiler = KJvmReplCompilerWithIdeServices() - return snippets.map { runRequest -> - with(runRequest) { - val newCompilationConfiguration = this.compilationConfiguration?.let { - ScriptCompilationConfiguration(compilationConfiguration, it) - } ?: compilationConfiguration - - val pos = SourceCode.Position(0, 0, cursor) - val codeLine = nextCodeLine(code) - val completionRes = if (doComplete && cursor != null) { - var res: ResultWithDiagnostics? - val timeMillis = measureTimeMillis { res = compiler.complete(codeLine, pos, newCompilationConfiguration) } - - loggingInfo?.complete?.writeValue(timeMillis) - - res!!.toList().filter { it.tail != "keyword" } - } else { - emptyList() - } - - val analysisResult = if (doErrorCheck) { - val codeLineForErrorCheck = nextCodeLine(code) - - var res: ReplAnalyzerResult? - val timeMillis = measureTimeMillis { - res = compiler.analyze(codeLineForErrorCheck, SourceCode.Position(0, 0), newCompilationConfiguration).valueOrNull() - } - - loggingInfo?.analyze?.writeValue(timeMillis) - - res - } else { - null - } ?: ReplAnalyzerResult() - - val errorsSequence = analysisResult[ReplAnalyzerResult.analysisDiagnostics]!! - val resultType = analysisResult[ReplAnalyzerResult.renderedResultType] - - if (doCompile) { - val codeLineForCompilation = nextCodeLine(code) - - val timeMillis = measureTimeMillis { compiler.compile(codeLineForCompilation, newCompilationConfiguration) } - - loggingInfo?.compile?.writeValue(timeMillis) - } - - ActualResult(completionRes, errorsSequence.toList(), resultType).asSuccess() - } - } - } - - private fun checkLists(index: Int, checkName: String, expected: List, actual: List, options: ExpectedOptions) { - when (options.mode) { - ComparisonType.EQUALS -> Assert.assertEquals( - "#$index ($checkName): Expected $expected, got $actual", - expected, - actual - ) - ComparisonType.INCLUDES -> Assert.assertTrue( - "#$index ($checkName): Expected $actual to include $expected", - actual.containsAll(expected) - ) - ComparisonType.COMPARE_SIZE -> Assert.assertEquals( - "#$index ($checkName): Expected list size to be equal to ${options.size}, but was ${actual.size}", - options.size, - actual.size - ) - ComparisonType.DONT_CHECK -> { - } - } - } - - private suspend fun checkEvaluateInRepl( - compilationConfiguration: ScriptCompilationConfiguration, - testData: List> - ) { - val (snippets, expected) = testData.unzip() - val expectedIter = expected.iterator() - evaluateInRepl(compilationConfiguration, snippets).forEachIndexed { index, res -> - when (res) { - is ResultWithDiagnostics.Failure -> Assert.fail("#$index: Expected result, got $res") - is ResultWithDiagnostics.Success -> { - val (expectedCompletions, expectedErrors, expectedResultType) = expectedIter.next() - val (completionsRes, errorsRes, resultType) = res.value - - checkLists(index, "completions", expectedCompletions.list, completionsRes, expectedCompletions) - val expectedErrorsWithPath = expectedErrors.list.map { - it.copy(sourcePath = errorsRes.firstOrNull()?.sourcePath) - } - checkLists(index, "errors", expectedErrorsWithPath, errorsRes, expectedErrors) - assertEquals("Analysis result types are different", expectedResultType, resultType) - } - } - } - } - +} + +// Artificial split into several testsuites, to speed up parallel testing +class IdeServicesLongRunningTest1 : TestCase() { + @Test + fun testLongRunningCompletion() = test { + // This test normally completes in about 15-25s + // Log should show slow _linear_ compilation/completion time growth + + val compileWriter = System.out.writer() // FileWriter("$csvDir/compilations.csv") + val completeWriter = System.out.writer() // FileWriter("$csvDir/completions.csv") + + for (i in 1..230) { + run(longCompilationRun(compileWriter, i)) + run(longCompletionRun(completeWriter, i)) + } + } +} + +// Artificial split into several testsuites, to speed up parallel testing +class IdeServicesLongRunningTest2 : TestCase() { + @Test + fun testLongRunningCompilation() = test { + // This test normally completes in about 10-20s + + val writer = System.out.writer() + for (i in 1..500) { + run(longCompilationRun(writer, i)) + } + } +} + +@Suppress("unused") +class TestReceiver1(val xyz1: Int = 42) + +@Suppress("unused") +class TestReceiver2(val xyz2: Int = 42) + +private fun longCompilationRun(writer: Writer, i: Int, conf: ScriptCompilationConfiguration? = null): TestRunConfigurator { + return { + conf?.let { + compilationConfiguration = it + } + + code = """ + val dataFrame = mapOf("x" to 45) + val e = "str" + """.trimIndent() + doCompile + + loggingInfo = CSVLoggingInfo(compile = CSVLoggingInfoItem(writer, i, "compile;")) + } +} + +private fun longCompletionRun(writer: Writer, i: Int, conf: ScriptCompilationConfiguration? = null): TestRunConfigurator { + return { + conf?.let { + compilationConfiguration = it + } + + code = """ + val x = mapOf("a" to dataFrame., "b" to 12, e to 42) + """.trimIndent() + cursor = 31 + + doComplete + expect { + completions.mode = ComparisonType.INCLUDES + addCompletion("entries", "entries", "Set>", "property") + } + + loggingInfo = CSVLoggingInfo(complete = CSVLoggingInfoItem(writer, i, "complete;")) + } } diff --git a/plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/test_util/testConfiguration.kt b/plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/test_util/testConfiguration.kt new file mode 100644 index 00000000000..9394381c0fc --- /dev/null +++ b/plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/test_util/testConfiguration.kt @@ -0,0 +1,293 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.scripting.ide_services.test_util + +import junit.framework.TestCase +import kotlinx.coroutines.runBlocking +import org.jetbrains.kotlin.scripting.ide_services.compiler.KJvmReplCompilerWithIdeServices +import org.junit.Assert +import java.io.Writer +import java.util.concurrent.atomic.AtomicInteger +import kotlin.reflect.KProperty +import kotlin.reflect.KProperty0 +import kotlin.script.experimental.api.* +import kotlin.system.measureTimeMillis + +class TestConf { + private val runs = mutableListOf() + + fun run(setup: (Run).() -> Unit) { + val r = Run() + r.setup() + runs.add(r) + } + + fun collect() = runs.map { it.collect() } + + class Run { + private var _doCompile = false + val doCompile: Unit + get() { + _doCompile = true + } + + private var _doComplete = false + val doComplete: Unit + get() { + _doComplete = true + } + + private var _doErrorCheck = false + val doErrorCheck: Unit + get() { + _doErrorCheck = true + } + + var cursor: Int? = null + var compilationConfiguration: ScriptCompilationConfiguration? = null + var code: String = "" + private var _expected: Expected = Expected(this) + + var loggingInfo: CSVLoggingInfo? = null + + fun expect(setup: (Expected).() -> Unit) { + _expected = Expected(this) + _expected.setup() + } + + fun collect(): Pair { + return RunRequest( + cursor, + code, + _doCompile, + _doComplete, + _doErrorCheck, + compilationConfiguration, + loggingInfo + ) to _expected.collect() + } + + class Expected(private val run: Run) { + val completions = ExpectedList(run::doComplete) + fun addCompletion(text: String, displayText: String, tail: String, icon: String) { + completions.add(SourceCodeCompletionVariant(text, displayText, tail, icon)) + } + + val errors = ExpectedList(run::doErrorCheck) + fun addError(startLine: Int, startCol: Int, endLine: Int, endCol: Int, message: String, severity: String) { + errors.add( + ScriptDiagnostic( + ScriptDiagnostic.unspecifiedError, + message, + ScriptDiagnostic.Severity.valueOf(severity), + location = SourceCode.Location( + SourceCode.Position(startLine, startCol), + SourceCode.Position(endLine, endCol) + ) + ) + ) + } + + var resultType: String? by ExpectedNullableVar(run::doErrorCheck) + + fun collect(): ExpectedResult { + return ExpectedResult(completions, errors, resultType) + } + } + + } +} + +fun test(setup: (TestConf).() -> Unit) { + val test = TestConf() + test.setup() + runBlocking { checkEvaluateInRepl(simpleScriptCompilationConfiguration, test.collect()) } +} + +enum class ComparisonType { + COMPARE_SIZE, INCLUDES, EQUALS, DONT_CHECK +} + +data class CSVLoggingInfoItem( + val writer: Writer, + val xValue: Int, + val prefix: String = "", +) { + fun writeValue(value: Any) { + writer.write("$prefix$xValue;$value\n") + writer.flush() + } +} + +data class CSVLoggingInfo( + val compile: CSVLoggingInfoItem? = null, + val complete: CSVLoggingInfoItem? = null, + val analyze: CSVLoggingInfoItem? = null, +) + +data class RunRequest( + val cursor: Int?, + val code: String, + val doCompile: Boolean, + val doComplete: Boolean, + val doErrorCheck: Boolean, + val compilationConfiguration: ScriptCompilationConfiguration?, + val loggingInfo: CSVLoggingInfo?, +) + +interface ExpectedOptions { + val mode: ComparisonType + val size: Int +} + +class ExpectedList(private val runProperty: KProperty0) : ExpectedOptions { + val list = mutableListOf() + + override var mode = ComparisonType.DONT_CHECK + override var size = 0 + set(value) { + if (mode == ComparisonType.DONT_CHECK) + mode = ComparisonType.COMPARE_SIZE + runProperty.get() + field = value + } + + fun add(elem: T) { + if (mode == ComparisonType.DONT_CHECK) + mode = ComparisonType.EQUALS + runProperty.get() + list.add(elem) + } +} + +class ExpectedNullableVar(private val runProperty: KProperty0) { + private var variable: T? = null + + operator fun getValue(thisRef: Any?, property: KProperty<*>): T? = variable + operator fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) { + runProperty.get() + variable = value + } +} + +data class ExpectedResult( + val completions: ExpectedList, + val errors: ExpectedList, + val resultType: String?, +) + +data class ActualResult( + val completions: List, + val errors: List, + val resultType: String?, +) + +private fun nextCodeLine(code: String, lineCounter: AtomicInteger): SourceCode = + SourceCodeTestImpl( + lineCounter.getAndIncrement(), + code + ) + +private suspend fun evaluateInRepl( + compilationConfiguration: ScriptCompilationConfiguration, + snippets: List, + lineCounter: AtomicInteger +): List> { + val compiler = KJvmReplCompilerWithIdeServices() + return snippets.map { runRequest -> + with(runRequest) { + val newCompilationConfiguration = this.compilationConfiguration?.let { + ScriptCompilationConfiguration(compilationConfiguration, it) + } ?: compilationConfiguration + + val pos = SourceCode.Position(0, 0, cursor) + val codeLine = nextCodeLine(code, lineCounter) + val completionRes = if (doComplete && cursor != null) { + var res: ResultWithDiagnostics? + val timeMillis = measureTimeMillis { res = compiler.complete(codeLine, pos, newCompilationConfiguration) } + + loggingInfo?.complete?.writeValue(timeMillis) + + res!!.toList().filter { it.tail != "keyword" } + } else { + emptyList() + } + + val analysisResult = if (doErrorCheck) { + val codeLineForErrorCheck = nextCodeLine(code, lineCounter) + + var res: ReplAnalyzerResult? + val timeMillis = measureTimeMillis { + res = compiler.analyze(codeLineForErrorCheck, SourceCode.Position(0, 0), newCompilationConfiguration).valueOrNull() + } + + loggingInfo?.analyze?.writeValue(timeMillis) + + res + } else { + null + } ?: ReplAnalyzerResult() + + val errorsSequence = analysisResult[ReplAnalyzerResult.analysisDiagnostics]!! + val resultType = analysisResult[ReplAnalyzerResult.renderedResultType] + + if (doCompile) { + val codeLineForCompilation = nextCodeLine(code, lineCounter) + + val timeMillis = measureTimeMillis { compiler.compile(codeLineForCompilation, newCompilationConfiguration) } + + loggingInfo?.compile?.writeValue(timeMillis) + } + + ActualResult(completionRes, errorsSequence.toList(), resultType).asSuccess() + } + } +} + +private fun checkLists(index: Int, checkName: String, expected: List, actual: List, options: ExpectedOptions) { + when (options.mode) { + ComparisonType.EQUALS -> Assert.assertEquals( + "#$index ($checkName): Expected $expected, got $actual", + expected, + actual + ) + ComparisonType.INCLUDES -> Assert.assertTrue( + "#$index ($checkName): Expected $actual to include $expected", + actual.containsAll(expected) + ) + ComparisonType.COMPARE_SIZE -> Assert.assertEquals( + "#$index ($checkName): Expected list size to be equal to ${options.size}, but was ${actual.size}", + options.size, + actual.size + ) + ComparisonType.DONT_CHECK -> { + } + } +} + +private suspend fun checkEvaluateInRepl( + compilationConfiguration: ScriptCompilationConfiguration, + testData: List> +) { + val (snippets, expected) = testData.unzip() + val expectedIter = expected.iterator() + evaluateInRepl(compilationConfiguration, snippets, AtomicInteger()).forEachIndexed { index, res -> + when (res) { + is ResultWithDiagnostics.Failure -> Assert.fail("#$index: Expected result, got $res") + is ResultWithDiagnostics.Success -> { + val (expectedCompletions, expectedErrors, expectedResultType) = expectedIter.next() + val (completionsRes, errorsRes, resultType) = res.value + + checkLists(index, "completions", expectedCompletions.list, completionsRes, expectedCompletions) + val expectedErrorsWithPath = expectedErrors.list.map { + it.copy(sourcePath = errorsRes.firstOrNull()?.sourcePath) + } + checkLists(index, "errors", expectedErrorsWithPath, errorsRes, expectedErrors) + TestCase.assertEquals("Analysis result types are different", expectedResultType, resultType) + } + } + } +}