Refactor REPL IDE services testing configuration and add new tests

plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/JvmReplTest.kt

@@ -270,9 +270,7 @@ class JvmIdeServicesTest : TestCase() {
     }
 }
 
-// Artificial split into several testsuites, to speed up parallel testing
+class LegacyReplTestLong : TestCase() {
-class LegacyReplTestLong1 : TestCase() {
-
     fun test256Evals() {
         JvmTestRepl()
             .use { repl ->
@@ -302,10 +300,6 @@ class LegacyReplTestLong1 : TestCase() {
                 assertEquals(evaluated.toString(), evals, (evaluated?.result as ResultValue.Value?)?.value)
             }
     }
-}
-
-// Artificial split into several testsuites, to speed up parallel testing
-class LegacyReplTestLong2 : TestCase() {
 
     fun testReplSlowdownKt22740() {
         JvmTestRepl()

plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/ReplCompletionAndErrorsAnalysisTest.kt

@@ -6,23 +6,21 @@
 package org.jetbrains.kotlin.scripting.ide_services
 
 import junit.framework.TestCase
-import kotlinx.coroutines.runBlocking
-import org.jetbrains.kotlin.scripting.ide_services.compiler.KJvmReplCompilerWithIdeServices
 import org.jetbrains.kotlin.scripting.ide_services.compiler.completion
 import org.jetbrains.kotlin.scripting.ide_services.compiler.filterOutShadowedDescriptors
 import org.jetbrains.kotlin.scripting.ide_services.compiler.nameFilter
-import org.jetbrains.kotlin.scripting.ide_services.test_util.SourceCodeTestImpl
+import org.jetbrains.kotlin.scripting.ide_services.test_util.*
-import org.jetbrains.kotlin.scripting.ide_services.test_util.simpleScriptCompilationConfiguration
+import org.jetbrains.kotlin.scripting.resolve.skipExtensionsResolutionForImplicits
-import org.jetbrains.kotlin.scripting.ide_services.test_util.toList
+import org.jetbrains.kotlin.scripting.resolve.skipExtensionsResolutionForImplicitsExceptInnermost
-import org.junit.Assert
 import org.junit.Ignore
 import org.junit.Test
 import java.io.Writer
-import java.util.concurrent.atomic.AtomicInteger
-import kotlin.reflect.KProperty
-import kotlin.reflect.KProperty0
 import kotlin.script.experimental.api.*
-import kotlin.system.measureTimeMillis
+import kotlin.script.experimental.jvm.jvm
+import kotlin.script.experimental.jvm.updateClasspath
+import kotlin.script.experimental.jvm.util.classpathFromClass
+
+typealias TestRunConfigurator = (TestConf.Run).() -> Unit
 
 class ReplCompletionAndErrorsAnalysisTest : TestCase() {
     @Test
@@ -293,43 +291,6 @@ class ReplCompletionAndErrorsAnalysisTest : TestCase() {
             }
     }
 
-    @Test
-    fun testLongRunningCompletion() = test {
-        // This test normally completes in about half a minute
-        // Log should show slow _linear_ compilation/completion time growth
-
-        val compileWriter = System.out.writer() // FileWriter("$csvDir/compilations.csv")
-        val completeWriter = System.out.writer() // FileWriter("$csvDir/completions.csv")
-
-        for (i in 1..230) {
-            run {
-                code = """
-                    val dataFrame = mapOf("x" to 45)
-                    val e = "str"
-                """.trimIndent()
-                doCompile
-
-                loggingInfo = CSVLoggingInfo(compile = CSVLoggingInfoItem(compileWriter, i, "compile;"))
-            }
-
-            run {
-                code = """
-                    val x = mapOf("a" to dataFrame., "b" to 12, e to 42)
-                """.trimIndent()
-                cursor = 31
-
-                doComplete
-                expect {
-                    completions.mode = ComparisonType.INCLUDES
-                    addCompletion("entries", "entries", "Set<Map.Entry<String, Int>>", "property")
-                }
-
-                loggingInfo = CSVLoggingInfo(complete = CSVLoggingInfoItem(completeWriter, i, "complete;"))
-            }
-
-        }
-    }
-
     @Test
     fun testImplicitExtensions() = test {
         run {
@@ -371,7 +332,43 @@ class ReplCompletionAndErrorsAnalysisTest : TestCase() {
         run(setupDefaultImportsCompletionRun)
     }
 
-    private val setupDefaultImportsCompletionRun: (TestConf.Run).() -> Unit = {
+    @Test
+    fun testLongRunningCompilationWithReceiver() = test {
+        // This test normally completes in about 8-13s
+        // Removing skip* configuration parameters should slow down the test (2-3 times)
+
+        val conf = ScriptCompilationConfiguration {
+            jvm {
+                updateClasspath(classpathFromClass<TestReceiver1>())
+            }
+            implicitReceivers(TestReceiver1::class, TestReceiver2::class)
+            skipExtensionsResolutionForImplicits(KotlinType(TestReceiver1::class))
+            skipExtensionsResolutionForImplicitsExceptInnermost(KotlinType(TestReceiver2::class))
+        }
+
+        val writer = System.out.writer()
+        for (i in 1..200) {
+            run(longCompilationRun(writer, i, conf))
+            run {
+                compilationConfiguration = conf
+
+                code = """
+                    val x = xyz
+                """.trimIndent()
+                cursor = 11
+
+                expect {
+                    completions.mode = ComparisonType.EQUALS
+                    addCompletion("xyz1", "xyz1", "Int", "property")
+                    addCompletion("xyz2", "xyz2", "Int", "property")
+                }
+
+                loggingInfo = CSVLoggingInfo(complete = CSVLoggingInfoItem(writer, i, "complete;"))
+            }
+        }
+    }
+
+    private val setupDefaultImportsCompletionRun: TestRunConfigurator = {
         compilationConfiguration = ScriptCompilationConfiguration {
             defaultImports(listOf("kotlin.math.atan"))
         }
@@ -384,283 +381,78 @@ class ReplCompletionAndErrorsAnalysisTest : TestCase() {
             addCompletion("atan(", "atan(Double)", "Double", "method")
         }
     }
-
+}
-    class TestConf {
+
-        private val runs = mutableListOf<Run>()
+// Artificial split into several testsuites, to speed up parallel testing
-
+class IdeServicesLongRunningTest1 : TestCase() {
-        fun run(setup: (Run).() -> Unit) {
+    @Test
-            val r = Run()
+    fun testLongRunningCompletion() = test {
-            r.setup()
+        // This test normally completes in about 15-25s
-            runs.add(r)
+        // Log should show slow _linear_ compilation/completion time growth
-        }
+
-
+        val compileWriter = System.out.writer() // FileWriter("$csvDir/compilations.csv")
-        fun collect() = runs.map { it.collect() }
+        val completeWriter = System.out.writer() // FileWriter("$csvDir/completions.csv")
-
+
-        class Run {
+        for (i in 1..230) {
-            private var _doCompile = false
+            run(longCompilationRun(compileWriter, i))
-            val doCompile: Unit
+            run(longCompletionRun(completeWriter, i))
-                get() {
+        }
-                    _doCompile = true
+    }
-                }
+}
-
+
-            private var _doComplete = false
+// Artificial split into several testsuites, to speed up parallel testing
-            val doComplete: Unit
+class IdeServicesLongRunningTest2 : TestCase() {
-                get() {
+    @Test
-                    _doComplete = true
+    fun testLongRunningCompilation() = test {
-                }
+        // This test normally completes in about 10-20s
-
+
-            private var _doErrorCheck = false
+        val writer = System.out.writer()
-            val doErrorCheck: Unit
+        for (i in 1..500) {
-                get() {
+            run(longCompilationRun(writer, i))
-                    _doErrorCheck = true
+        }
-                }
+    }
-
+}
-            var cursor: Int? = null
+
-            var compilationConfiguration: ScriptCompilationConfiguration? = null
+@Suppress("unused")
-            var code: String = ""
+class TestReceiver1(val xyz1: Int = 42)
-            private var _expected: Expected = Expected(this)
+
-
+@Suppress("unused")
-            var loggingInfo: CSVLoggingInfo? = null
+class TestReceiver2(val xyz2: Int = 42)
-
+
-            fun expect(setup: (Expected).() -> Unit) {
+private fun longCompilationRun(writer: Writer, i: Int, conf: ScriptCompilationConfiguration? = null): TestRunConfigurator {
-                _expected = Expected(this)
+    return {
-                _expected.setup()
+        conf?.let {
-            }
+            compilationConfiguration = it
-
+        }
-            fun collect(): Pair<RunRequest, ExpectedResult> {
+
-                return RunRequest(
+        code = """
-                    cursor,
+            val dataFrame = mapOf("x" to 45)
-                    code,
+            val e = "str"
-                    _doCompile,
+        """.trimIndent()
-                    _doComplete,
+        doCompile
-                    _doErrorCheck,
+
-                    compilationConfiguration,
+        loggingInfo = CSVLoggingInfo(compile = CSVLoggingInfoItem(writer, i, "compile;"))
-                    loggingInfo
+    }
-                ) to _expected.collect()
+}
-            }
+
-
+private fun longCompletionRun(writer: Writer, i: Int, conf: ScriptCompilationConfiguration? = null): TestRunConfigurator {
-            class Expected(private val run: Run) {
+    return {
-                val completions = ExpectedList<SourceCodeCompletionVariant>(run::doComplete)
+        conf?.let {
-                fun addCompletion(text: String, displayText: String, tail: String, icon: String) {
+            compilationConfiguration = it
-                    completions.add(SourceCodeCompletionVariant(text, displayText, tail, icon))
+        }
-                }
+
-
+        code = """
-                val errors = ExpectedList<ScriptDiagnostic>(run::doErrorCheck)
+            val x = mapOf("a" to dataFrame., "b" to 12, e to 42)
-                fun addError(startLine: Int, startCol: Int, endLine: Int, endCol: Int, message: String, severity: String) {
+        """.trimIndent()
-                    errors.add(
+        cursor = 31
-                        ScriptDiagnostic(
+
-                            ScriptDiagnostic.unspecifiedError,
+        doComplete
-                            message,
+        expect {
-                            ScriptDiagnostic.Severity.valueOf(severity),
+            completions.mode = ComparisonType.INCLUDES
-                            location = SourceCode.Location(
+            addCompletion("entries", "entries", "Set<Map.Entry<String, Int>>", "property")
-                                SourceCode.Position(startLine, startCol),
+        }
-                                SourceCode.Position(endLine, endCol)
+
-                            )
+        loggingInfo = CSVLoggingInfo(complete = CSVLoggingInfoItem(writer, i, "complete;"))
-                        )
+    }
-                    )
-                }
-
-                var resultType: String? by ExpectedNullableVar(run::doErrorCheck)
-
-                fun collect(): ExpectedResult {
-                    return ExpectedResult(completions, errors, resultType)
-                }
-            }
-
-        }
-    }
-
-    private fun test(setup: (TestConf).() -> Unit) {
-        val test = TestConf()
-        test.setup()
-        runBlocking { checkEvaluateInRepl(simpleScriptCompilationConfiguration, test.collect()) }
-    }
-
-    enum class ComparisonType {
-        COMPARE_SIZE, INCLUDES, EQUALS, DONT_CHECK
-    }
-
-    data class CSVLoggingInfoItem(
-        val writer: Writer,
-        val xValue: Int,
-        val prefix: String = "",
-    ) {
-        fun writeValue(value: Any) {
-            writer.write("$prefix$xValue;$value\n")
-            writer.flush()
-        }
-    }
-
-    data class CSVLoggingInfo(
-        val compile: CSVLoggingInfoItem? = null,
-        val complete: CSVLoggingInfoItem? = null,
-        val analyze: CSVLoggingInfoItem? = null,
-    )
-
-    data class RunRequest(
-        val cursor: Int?,
-        val code: String,
-        val doCompile: Boolean,
-        val doComplete: Boolean,
-        val doErrorCheck: Boolean,
-        val compilationConfiguration: ScriptCompilationConfiguration?,
-        val loggingInfo: CSVLoggingInfo?,
-    )
-
-    interface ExpectedOptions {
-        val mode: ComparisonType
-        val size: Int
-    }
-
-    class ExpectedList<T>(private val runProperty: KProperty0<Unit>) : ExpectedOptions {
-        val list = mutableListOf<T>()
-
-        override var mode = ComparisonType.DONT_CHECK
-        override var size = 0
-            set(value) {
-                if (mode == ComparisonType.DONT_CHECK)
-                    mode = ComparisonType.COMPARE_SIZE
-                runProperty.get()
-                field = value
-            }
-
-        fun add(elem: T) {
-            if (mode == ComparisonType.DONT_CHECK)
-                mode = ComparisonType.EQUALS
-            runProperty.get()
-            list.add(elem)
-        }
-    }
-
-    class ExpectedNullableVar<T>(private val runProperty: KProperty0<Unit>) {
-        private var variable: T? = null
-
-        operator fun getValue(thisRef: Any?, property: KProperty<*>): T? = variable
-        operator fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) {
-            runProperty.get()
-            variable = value
-        }
-    }
-
-    data class ExpectedResult(
-        val completions: ExpectedList<SourceCodeCompletionVariant>,
-        val errors: ExpectedList<ScriptDiagnostic>,
-        val resultType: String?,
-    )
-
-    data class ActualResult(
-        val completions: List<SourceCodeCompletionVariant>,
-        val errors: List<ScriptDiagnostic>,
-        val resultType: String?,
-    )
-
-    private val currentLineCounter = AtomicInteger()
-
-    private fun nextCodeLine(code: String): SourceCode =
-        SourceCodeTestImpl(
-            currentLineCounter.getAndIncrement(),
-            code
-        )
-
-    private suspend fun evaluateInRepl(
-        compilationConfiguration: ScriptCompilationConfiguration,
-        snippets: List<RunRequest>
-    ): List<ResultWithDiagnostics<ActualResult>> {
-        val compiler = KJvmReplCompilerWithIdeServices()
-        return snippets.map { runRequest ->
-            with(runRequest) {
-                val newCompilationConfiguration = this.compilationConfiguration?.let {
-                    ScriptCompilationConfiguration(compilationConfiguration, it)
-                } ?: compilationConfiguration
-
-                val pos = SourceCode.Position(0, 0, cursor)
-                val codeLine = nextCodeLine(code)
-                val completionRes = if (doComplete && cursor != null) {
-                    var res: ResultWithDiagnostics<ReplCompletionResult>?
-                    val timeMillis = measureTimeMillis { res = compiler.complete(codeLine, pos, newCompilationConfiguration) }
-
-                    loggingInfo?.complete?.writeValue(timeMillis)
-
-                    res!!.toList().filter { it.tail != "keyword" }
-                } else {
-                    emptyList()
-                }
-
-                val analysisResult = if (doErrorCheck) {
-                    val codeLineForErrorCheck = nextCodeLine(code)
-
-                    var res: ReplAnalyzerResult?
-                    val timeMillis = measureTimeMillis {
-                        res = compiler.analyze(codeLineForErrorCheck, SourceCode.Position(0, 0), newCompilationConfiguration).valueOrNull()
-                    }
-
-                    loggingInfo?.analyze?.writeValue(timeMillis)
-
-                    res
-                } else {
-                    null
-                } ?: ReplAnalyzerResult()
-
-                val errorsSequence = analysisResult[ReplAnalyzerResult.analysisDiagnostics]!!
-                val resultType = analysisResult[ReplAnalyzerResult.renderedResultType]
-
-                if (doCompile) {
-                    val codeLineForCompilation = nextCodeLine(code)
-
-                    val timeMillis = measureTimeMillis { compiler.compile(codeLineForCompilation, newCompilationConfiguration) }
-
-                    loggingInfo?.compile?.writeValue(timeMillis)
-                }
-
-                ActualResult(completionRes, errorsSequence.toList(), resultType).asSuccess()
-            }
-        }
-    }
-
-    private fun <T> checkLists(index: Int, checkName: String, expected: List<T>, actual: List<T>, options: ExpectedOptions) {
-        when (options.mode) {
-            ComparisonType.EQUALS -> Assert.assertEquals(
-                "#$index ($checkName): Expected $expected, got $actual",
-                expected,
-                actual
-            )
-            ComparisonType.INCLUDES -> Assert.assertTrue(
-                "#$index ($checkName): Expected $actual to include $expected",
-                actual.containsAll(expected)
-            )
-            ComparisonType.COMPARE_SIZE -> Assert.assertEquals(
-                "#$index ($checkName): Expected list size to be equal to ${options.size}, but was ${actual.size}",
-                options.size,
-                actual.size
-            )
-            ComparisonType.DONT_CHECK -> {
-            }
-        }
-    }
-
-    private suspend fun checkEvaluateInRepl(
-        compilationConfiguration: ScriptCompilationConfiguration,
-        testData: List<Pair<RunRequest, ExpectedResult>>
-    ) {
-        val (snippets, expected) = testData.unzip()
-        val expectedIter = expected.iterator()
-        evaluateInRepl(compilationConfiguration, snippets).forEachIndexed { index, res ->
-            when (res) {
-                is ResultWithDiagnostics.Failure -> Assert.fail("#$index: Expected result, got $res")
-                is ResultWithDiagnostics.Success -> {
-                    val (expectedCompletions, expectedErrors, expectedResultType) = expectedIter.next()
-                    val (completionsRes, errorsRes, resultType) = res.value
-
-                    checkLists(index, "completions", expectedCompletions.list, completionsRes, expectedCompletions)
-                    val expectedErrorsWithPath = expectedErrors.list.map {
-                        it.copy(sourcePath = errorsRes.firstOrNull()?.sourcePath)
-                    }
-                    checkLists(index, "errors", expectedErrorsWithPath, errorsRes, expectedErrors)
-                    assertEquals("Analysis result types are different", expectedResultType, resultType)
-                }
-            }
-        }
-    }
-
 }
 

plugins/scripting/scripting-ide-services-test/test/org/jetbrains/kotlin/scripting/ide_services/test_util/testConfiguration.kt

@@ -0,0 +1,293 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.scripting.ide_services.test_util
+
+import junit.framework.TestCase
+import kotlinx.coroutines.runBlocking
+import org.jetbrains.kotlin.scripting.ide_services.compiler.KJvmReplCompilerWithIdeServices
+import org.junit.Assert
+import java.io.Writer
+import java.util.concurrent.atomic.AtomicInteger
+import kotlin.reflect.KProperty
+import kotlin.reflect.KProperty0
+import kotlin.script.experimental.api.*
+import kotlin.system.measureTimeMillis
+
+class TestConf {
+    private val runs = mutableListOf<Run>()
+
+    fun run(setup: (Run).() -> Unit) {
+        val r = Run()
+        r.setup()
+        runs.add(r)
+    }
+
+    fun collect() = runs.map { it.collect() }
+
+    class Run {
+        private var _doCompile = false
+        val doCompile: Unit
+            get() {
+                _doCompile = true
+            }
+
+        private var _doComplete = false
+        val doComplete: Unit
+            get() {
+                _doComplete = true
+            }
+
+        private var _doErrorCheck = false
+        val doErrorCheck: Unit
+            get() {
+                _doErrorCheck = true
+            }
+
+        var cursor: Int? = null
+        var compilationConfiguration: ScriptCompilationConfiguration? = null
+        var code: String = ""
+        private var _expected: Expected = Expected(this)
+
+        var loggingInfo: CSVLoggingInfo? = null
+
+        fun expect(setup: (Expected).() -> Unit) {
+            _expected = Expected(this)
+            _expected.setup()
+        }
+
+        fun collect(): Pair<RunRequest, ExpectedResult> {
+            return RunRequest(
+                cursor,
+                code,
+                _doCompile,
+                _doComplete,
+                _doErrorCheck,
+                compilationConfiguration,
+                loggingInfo
+            ) to _expected.collect()
+        }
+
+        class Expected(private val run: Run) {
+            val completions = ExpectedList<SourceCodeCompletionVariant>(run::doComplete)
+            fun addCompletion(text: String, displayText: String, tail: String, icon: String) {
+                completions.add(SourceCodeCompletionVariant(text, displayText, tail, icon))
+            }
+
+            val errors = ExpectedList<ScriptDiagnostic>(run::doErrorCheck)
+            fun addError(startLine: Int, startCol: Int, endLine: Int, endCol: Int, message: String, severity: String) {
+                errors.add(
+                    ScriptDiagnostic(
+                        ScriptDiagnostic.unspecifiedError,
+                        message,
+                        ScriptDiagnostic.Severity.valueOf(severity),
+                        location = SourceCode.Location(
+                            SourceCode.Position(startLine, startCol),
+                            SourceCode.Position(endLine, endCol)
+                        )
+                    )
+                )
+            }
+
+            var resultType: String? by ExpectedNullableVar(run::doErrorCheck)
+
+            fun collect(): ExpectedResult {
+                return ExpectedResult(completions, errors, resultType)
+            }
+        }
+
+    }
+}
+
+fun test(setup: (TestConf).() -> Unit) {
+    val test = TestConf()
+    test.setup()
+    runBlocking { checkEvaluateInRepl(simpleScriptCompilationConfiguration, test.collect()) }
+}
+
+enum class ComparisonType {
+    COMPARE_SIZE, INCLUDES, EQUALS, DONT_CHECK
+}
+
+data class CSVLoggingInfoItem(
+    val writer: Writer,
+    val xValue: Int,
+    val prefix: String = "",
+) {
+    fun writeValue(value: Any) {
+        writer.write("$prefix$xValue;$value\n")
+        writer.flush()
+    }
+}
+
+data class CSVLoggingInfo(
+    val compile: CSVLoggingInfoItem? = null,
+    val complete: CSVLoggingInfoItem? = null,
+    val analyze: CSVLoggingInfoItem? = null,
+)
+
+data class RunRequest(
+    val cursor: Int?,
+    val code: String,
+    val doCompile: Boolean,
+    val doComplete: Boolean,
+    val doErrorCheck: Boolean,
+    val compilationConfiguration: ScriptCompilationConfiguration?,
+    val loggingInfo: CSVLoggingInfo?,
+)
+
+interface ExpectedOptions {
+    val mode: ComparisonType
+    val size: Int
+}
+
+class ExpectedList<T>(private val runProperty: KProperty0<Unit>) : ExpectedOptions {
+    val list = mutableListOf<T>()
+
+    override var mode = ComparisonType.DONT_CHECK
+    override var size = 0
+        set(value) {
+            if (mode == ComparisonType.DONT_CHECK)
+                mode = ComparisonType.COMPARE_SIZE
+            runProperty.get()
+            field = value
+        }
+
+    fun add(elem: T) {
+        if (mode == ComparisonType.DONT_CHECK)
+            mode = ComparisonType.EQUALS
+        runProperty.get()
+        list.add(elem)
+    }
+}
+
+class ExpectedNullableVar<T>(private val runProperty: KProperty0<Unit>) {
+    private var variable: T? = null
+
+    operator fun getValue(thisRef: Any?, property: KProperty<*>): T? = variable
+    operator fun setValue(thisRef: Any?, property: KProperty<*>, value: T?) {
+        runProperty.get()
+        variable = value
+    }
+}
+
+data class ExpectedResult(
+    val completions: ExpectedList<SourceCodeCompletionVariant>,
+    val errors: ExpectedList<ScriptDiagnostic>,
+    val resultType: String?,
+)
+
+data class ActualResult(
+    val completions: List<SourceCodeCompletionVariant>,
+    val errors: List<ScriptDiagnostic>,
+    val resultType: String?,
+)
+
+private fun nextCodeLine(code: String, lineCounter: AtomicInteger): SourceCode =
+    SourceCodeTestImpl(
+        lineCounter.getAndIncrement(),
+        code
+    )
+
+private suspend fun evaluateInRepl(
+    compilationConfiguration: ScriptCompilationConfiguration,
+    snippets: List<RunRequest>,
+    lineCounter: AtomicInteger
+): List<ResultWithDiagnostics<ActualResult>> {
+    val compiler = KJvmReplCompilerWithIdeServices()
+    return snippets.map { runRequest ->
+        with(runRequest) {
+            val newCompilationConfiguration = this.compilationConfiguration?.let {
+                ScriptCompilationConfiguration(compilationConfiguration, it)
+            } ?: compilationConfiguration
+
+            val pos = SourceCode.Position(0, 0, cursor)
+            val codeLine = nextCodeLine(code, lineCounter)
+            val completionRes = if (doComplete && cursor != null) {
+                var res: ResultWithDiagnostics<ReplCompletionResult>?
+                val timeMillis = measureTimeMillis { res = compiler.complete(codeLine, pos, newCompilationConfiguration) }
+
+                loggingInfo?.complete?.writeValue(timeMillis)
+
+                res!!.toList().filter { it.tail != "keyword" }
+            } else {
+                emptyList()
+            }
+
+            val analysisResult = if (doErrorCheck) {
+                val codeLineForErrorCheck = nextCodeLine(code, lineCounter)
+
+                var res: ReplAnalyzerResult?
+                val timeMillis = measureTimeMillis {
+                    res = compiler.analyze(codeLineForErrorCheck, SourceCode.Position(0, 0), newCompilationConfiguration).valueOrNull()
+                }
+
+                loggingInfo?.analyze?.writeValue(timeMillis)
+
+                res
+            } else {
+                null
+            } ?: ReplAnalyzerResult()
+
+            val errorsSequence = analysisResult[ReplAnalyzerResult.analysisDiagnostics]!!
+            val resultType = analysisResult[ReplAnalyzerResult.renderedResultType]
+
+            if (doCompile) {
+                val codeLineForCompilation = nextCodeLine(code, lineCounter)
+
+                val timeMillis = measureTimeMillis { compiler.compile(codeLineForCompilation, newCompilationConfiguration) }
+
+                loggingInfo?.compile?.writeValue(timeMillis)
+            }
+
+            ActualResult(completionRes, errorsSequence.toList(), resultType).asSuccess()
+        }
+    }
+}
+
+private fun <T> checkLists(index: Int, checkName: String, expected: List<T>, actual: List<T>, options: ExpectedOptions) {
+    when (options.mode) {
+        ComparisonType.EQUALS -> Assert.assertEquals(
+            "#$index ($checkName): Expected $expected, got $actual",
+            expected,
+            actual
+        )
+        ComparisonType.INCLUDES -> Assert.assertTrue(
+            "#$index ($checkName): Expected $actual to include $expected",
+            actual.containsAll(expected)
+        )
+        ComparisonType.COMPARE_SIZE -> Assert.assertEquals(
+            "#$index ($checkName): Expected list size to be equal to ${options.size}, but was ${actual.size}",
+            options.size,
+            actual.size
+        )
+        ComparisonType.DONT_CHECK -> {
+        }
+    }
+}
+
+private suspend fun checkEvaluateInRepl(
+    compilationConfiguration: ScriptCompilationConfiguration,
+    testData: List<Pair<RunRequest, ExpectedResult>>
+) {
+    val (snippets, expected) = testData.unzip()
+    val expectedIter = expected.iterator()
+    evaluateInRepl(compilationConfiguration, snippets, AtomicInteger()).forEachIndexed { index, res ->
+        when (res) {
+            is ResultWithDiagnostics.Failure -> Assert.fail("#$index: Expected result, got $res")
+            is ResultWithDiagnostics.Success -> {
+                val (expectedCompletions, expectedErrors, expectedResultType) = expectedIter.next()
+                val (completionsRes, errorsRes, resultType) = res.value
+
+                checkLists(index, "completions", expectedCompletions.list, completionsRes, expectedCompletions)
+                val expectedErrorsWithPath = expectedErrors.list.map {
+                    it.copy(sourcePath = errorsRes.firstOrNull()?.sourcePath)
+                }
+                checkLists(index, "errors", expectedErrorsWithPath, errorsRes, expectedErrors)
+                TestCase.assertEquals("Analysis result types are different", expectedResultType, resultType)
+            }
+        }
+    }
+}