[Native][tests] Support JUnit5 dynamic tests

2021-11-18 22:30:03 +03:00
parent b5daf78f8d
commit 96067b07d9
10 changed files with 497 additions and 118 deletions
@@ -1,5 +1,5 @@
-// FILE: main.kt
-package samples.regular_multifile_with_explicit_packages
+// FILE: a_b_c.kt
+package samples.regular_multifile_with_explicit_packages.a.b.c

 import kotlin.test.*

@@ -23,8 +23,8 @@ fun division () {
    assertEquals(42, 126 / 3)
 }

-// FILE: a.kt
-package samples.regular_multifile_with_explicit_packages.a
+// FILE: a_b_c_d.kt
+package samples.regular_multifile_with_explicit_packages.a.b.c.d

 import kotlin.test.*

@@ -48,8 +48,8 @@ fun division () {
    assertEquals(42, 126 / 3)
 }

-// FILE: b.kt
-package samples.regular_multifile_with_explicit_packages.b
+// FILE: a_b_c_d_e.kt
+package samples.regular_multifile_with_explicit_packages.a.b.c.d.e

 import kotlin.test.*

@@ -73,8 +73,58 @@ fun division () {
    assertEquals(42, 126 / 3)
 }

-// FILE: a_b.kt
-package samples.regular_multifile_with_explicit_packages.a.b
+// FILE: a_b_q_w_e.kt
+package samples.regular_multifile_with_explicit_packages.a.b.q.w.e
+
+import kotlin.test.*
+
+@Test
+fun addition() {
+    assertEquals(42, 40 + 2)
+}
+
+@Test
+fun multiplication () {
+    assertEquals(42, 21 * 2)
+}
+
+@Test
+fun subtraction () {
+    assertEquals(42, 50 - 8)
+}
+
+@Test
+fun division () {
+    assertEquals(42, 126 / 3)
+}
+
+// FILE: a_b_q_w_e_r.kt
+package samples.regular_multifile_with_explicit_packages.a.b.q.w.e.r
+
+import kotlin.test.*
+
+@Test
+fun addition() {
+    assertEquals(42, 40 + 2)
+}
+
+@Test
+fun multiplication () {
+    assertEquals(42, 21 * 2)
+}
+
+@Test
+fun subtraction () {
+    assertEquals(42, 50 - 8)
+}
+
+@Test
+fun division () {
+    assertEquals(42, 126 / 3)
+}
+
+// FILE: a_b_q_w_e_r_t_y.kt
+package samples.regular_multifile_with_explicit_packages.a.b.q.w.e.r.t.y

 import kotlin.test.*

@@ -6,9 +6,12 @@
 package org.jetbrains.kotlin.konan.blackboxtest

 import com.intellij.testFramework.TestDataFile
-import org.jetbrains.kotlin.konan.blackboxtest.support.NativeBlackBoxTestSupport
-import org.jetbrains.kotlin.konan.blackboxtest.support.TestRunProvider
+import org.jetbrains.kotlin.konan.blackboxtest.support.*
+import org.jetbrains.kotlin.konan.blackboxtest.support.util.TreeNode
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.getAbsoluteFile
+import org.junit.jupiter.api.DynamicContainer.dynamicContainer
+import org.junit.jupiter.api.DynamicNode
+import org.junit.jupiter.api.DynamicTest.dynamicTest
 import org.junit.jupiter.api.extension.ExtendWith
 import java.io.File

@@ -28,10 +31,40 @@ abstract class AbstractNativeBlackBoxTest {
    fun register(@TestDataFile testDataFilePath: String, sourceTransformer: (String) -> String) =
        register(testDataFilePath, listOf(sourceTransformer))

-    fun runTest(@TestDataFile testDataFilePath: String): Unit = with(testRunProvider) {
+    /**
+     * Run JUnit test.
+     *
+     * This function should be called from a method annotated with [org.junit.jupiter.api.Test].
+     */
+    fun runTest(@TestDataFile testDataFilePath: String) {
        val testDataFile = getAbsoluteFile(testDataFilePath)
-        val testRun = getSingleTestRun(testDataFile)
-        val testRunner = createRunner(testRun)
+        val testRun = testRunProvider.getSingleTestRun(testDataFile)
+        runTest(testRun)
+    }
+
+    /**
+     * Run JUnit dynamic test.
+     *
+     * This function should be called from a method annotated with [org.junit.jupiter.api.TestFactory].
+     */
+    fun dynamicTest(@TestDataFile testDataFilePath: String): Collection<DynamicNode> {
+        val testDataFile = getAbsoluteFile(testDataFilePath)
+        val rootTestRunNode = testRunProvider.getTestRuns(testDataFile)
+        return buildJUnitDynamicNodes(rootTestRunNode)
+    }
+
+    private fun buildJUnitDynamicNodes(testRunNode: TreeNode<TestRun>): Collection<DynamicNode> = buildList {
+        testRunNode.items.mapTo(this) { testRun ->
+            dynamicTest(testRun.displayName) { runTest(testRun) }
+        }
+
+        testRunNode.children.mapTo(this) { childTestRunNode ->
+            dynamicContainer(childTestRunNode.packageSegment, buildJUnitDynamicNodes(childTestRunNode))
+        }
+    }
+
+    private fun runTest(testRun: TestRun) {
+        val testRunner = testRunProvider.createRunner(testRun)
        testRunner.run()
    }
 }
@@ -3,17 +3,19 @@
 * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
 */

+@file:Suppress("KDocUnresolvedReference")
+
 package org.jetbrains.kotlin.konan.blackboxtest.support

 import org.jetbrains.kotlin.konan.blackboxtest.support.TestModule.Companion.allDependencies
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.*
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
-import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertEquals
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
 import java.io.File

 internal typealias PackageFQN = String
+internal typealias FunctionName = String

 /**
 * Helps to track the origin of every [TestCase], [TestCompilation] or [TestExecutable]. Used for issue reporting purposes.
@@ -24,6 +26,11 @@ internal interface TestOrigin {
    }
 }

+/**
+ * Represents a single test function (i.e. a function annotated with [kotlin.test.Test]) inside of a [TestFile].
+ */
+internal data class TestFunction(val packageName: PackageFQN, val functionName: FunctionName)
+
 /**
 * Represents a single file that will be supplied to the compiler.
 */
@@ -37,11 +44,18 @@ internal class TestFile<M : TestModule> private constructor(
        class Uncommitted(var text: String) : State
    }

-    fun update(transformation: (String) -> String) {
-        when (val state = state) {
-            is State.Uncommitted -> state.text = transformation(state.text)
+    private val uncommittedState: State.Uncommitted
+        get() = when (val state = state) {
+            is State.Uncommitted -> state
            is State.Committed -> fail { "File $location is already committed." }
        }
+
+    val text: String
+        get() = uncommittedState.text
+
+    fun update(transformation: (String) -> String) {
+        val uncommittedState = uncommittedState
+        uncommittedState.text = transformation(uncommittedState.text)
    }

    // An optimization to release the memory occupied by numerous file texts.
@@ -156,9 +170,27 @@ internal class TestCase(
    val origin: TestOrigin.SingleTestDataFile,
    val nominalPackageName: PackageFQN,
    val expectedOutputDataFile: File?,
-    val extras: StandaloneNoTestRunnerExtras? = null
+    val extras: Extras
 ) {
-    // The set of module that have no incoming dependency arcs.
+    sealed interface Extras
+    class NoTestRunnerExtras(val entryPoint: String, val inputDataFile: File?) : Extras
+    class WithTestRunnerExtras(val testFunctions: Collection<TestFunction>) : Extras {
+        companion object {
+            val EMPTY = WithTestRunnerExtras(emptyList())
+        }
+    }
+
+    init {
+        when (kind) {
+            TestKind.STANDALONE_NO_TR -> assertTrue(extras is NoTestRunnerExtras)
+            TestKind.REGULAR, TestKind.STANDALONE -> assertTrue(extras is WithTestRunnerExtras)
+        }
+    }
+
+    inline fun <reified T : Extras> extras(): T = extras as T
+    inline fun <reified T : Extras> safeExtras(): T? = extras as? T
+
+    // The set of modules that have no incoming dependency arcs.
    val rootModules: Set<TestModule.Exclusive> by lazy {
        val allModules = hashSetOf<TestModule>()
        modules.forEach { module ->
@@ -182,12 +214,6 @@ internal class TestCase(
        rootModules as Set<TestModule.Exclusive>
    }

-    class StandaloneNoTestRunnerExtras(val entryPoint: String, val inputDataFile: File?)
-
-    init {
-        assertEquals(extras != null, kind == TestKind.STANDALONE_NO_TR)
-    }
-
    fun initialize(findSharedModule: ((moduleName: String) -> TestModule.Shared?)?) {
        // Check that there are no duplicated files among different modules.
        val duplicatedFiles = modules.flatMap { it.files }.groupingBy { it }.eachCount().filterValues { it > 1 }.keys
@@ -10,6 +10,7 @@ import org.jetbrains.kotlin.cli.common.messages.*
 import org.jetbrains.kotlin.compilerRunner.OutputItemsCollectorImpl
 import org.jetbrains.kotlin.compilerRunner.processCompilerOutput
 import org.jetbrains.kotlin.config.Services
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.NoTestRunnerExtras
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCompilation.Companion.resultingArtifactPath
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCompilationResult.Companion.assertSuccess
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestModule.Companion.allDependencies
@@ -42,7 +43,7 @@ internal class TestCompilationFactory(private val environment: TestEnvironment)
        val friends = rootModules.flatMapToSet { it.allFriends }.map { moduleToKlib(it, freeCompilerArgs) }

        return cachedCompilations.computeIfAbsent(cacheKey) {
-            val entryPoint = testCases.singleOrNull()?.extras?.entryPoint
+            val entryPoint = testCases.singleOrNull()?.safeExtras<NoTestRunnerExtras>()?.entryPoint

            TestCompilationImpl(
                environment = environment,
@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.konan.blackboxtest.support

+import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
 import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
 import java.io.File

@@ -14,6 +15,7 @@ internal class TestExecutable(
 )

 internal class TestRun(
+    val displayName: String,
    val executable: TestExecutable,
    val runParameters: List<TestRunParameter>,
    val origin: TestOrigin.SingleTestDataFile
@@ -23,26 +25,47 @@ internal sealed interface TestRunParameter {
    fun applyTo(programArgs: MutableList<String>)

    sealed class WithFilter : TestRunParameter {
-        protected abstract val wildcard: String
        abstract fun testMatches(testName: String): Boolean
+    }

-        final override fun applyTo(programArgs: MutableList<String>) {
-            programArgs += "--ktest_filter=$wildcard"
+    class WithPackageFilter(packageName: PackageFQN) : WithFilter() {
+        init {
+            assertTrue(packageName.isNotEmpty())
+        }
+
+        private val packagePrefix = "$packageName."
+
+        override fun applyTo(programArgs: MutableList<String>) {
+            programArgs += "--ktest_filter=$packagePrefix*"
+        }
+
+        override fun testMatches(testName: String) = testName.startsWith(packagePrefix)
+    }
+
+    class WithFunctionFilter(val testFunction: TestFunction) : WithFilter() {
+        private val packagePrefix = if (testFunction.packageName.isNotEmpty()) "${testFunction.packageName}." else ""
+
+        override fun applyTo(programArgs: MutableList<String>) {
+            programArgs += "--ktest_regex_filter=${packagePrefix.replace(".", "\\.")}([^\\.]+)\\.${testFunction.functionName}"
+        }
+
+        override fun testMatches(testName: String): Boolean {
+            val remainder = if (packagePrefix.isNotEmpty()) {
+                if (!testName.startsWith(packagePrefix)) return false
+                testName.substringAfter(packagePrefix)
+            } else
+                testName
+
+            val suffix = remainder
+                .split('.')
+                .takeIf { it.size == 2 }
+                ?.last()
+                ?: return false
+
+            return suffix == testFunction.functionName
        }
    }

-    class WithPackageFilter(private val packageFQN: PackageFQN) : WithFilter() {
-        override val wildcard get() = "$packageFQN.*"
-        override fun testMatches(testName: String) = testName.startsWith("$packageFQN.")
-    }
-
-/*
-    class WithFunctionFilter(private val functionFQN: String) : WithFilter() {
-        override val wildcard get() = functionFQN
-        override fun testMatches(testName: String) = testName == functionFQN
-    }
-*/
-
    object WithGTestLogger : TestRunParameter {
        override fun applyTo(programArgs: MutableList<String>) {
            programArgs += "--ktest_logger=GTEST"
@@ -3,14 +3,20 @@
 * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
 */

+@file:Suppress("KDocUnresolvedReference")
+
 package org.jetbrains.kotlin.konan.blackboxtest.support

 import com.intellij.openapi.util.Disposer
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.NoTestRunnerExtras
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestCompilationResult.Companion.assertSuccess
 import org.jetbrains.kotlin.konan.blackboxtest.support.group.TestCaseGroupProvider
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.AbstractRunner
 import org.jetbrains.kotlin.konan.blackboxtest.support.runner.LocalTestRunner
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.ThreadSafeCache
+import org.jetbrains.kotlin.konan.blackboxtest.support.util.TreeNode
+import org.jetbrains.kotlin.konan.blackboxtest.support.util.buildTree
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
 import org.junit.jupiter.api.Assumptions.assumeTrue
@@ -28,7 +34,86 @@ internal class TestRunProvider(
        testCaseGroupProvider.setPreprocessors(testDataFile, sourceTransformers)
    }

-    fun getSingleTestRun(testDataFile: File): TestRun {
+    /**
+     * Produces a single [TestRun] per testData file.
+     *
+     * If testData file contains multiple functions annotated with [kotlin.test.Test], then all these functions will be executed
+     * in one shot. If either function will fail, the whole JUnit test will be considered as failed.
+     *
+     * Example:
+     *   //+++ testData file (foo.kt): +++//
+     *   @kotlin.test.Test
+     *   fun one() { /* ... */ }
+     *
+     *   @kotlin.test.Test
+     *   fun two() { /* ... */ }
+     *
+     *   //+++ generated JUnit test suite: +++//
+     *   public class MyTestSuiteGenerated {
+     *       @org.junit.jupiter.api.Test
+     *       @org.jetbrains.kotlin.test.TestMetadata("foo.kt")
+     *       public void testFoo() {
+     *           // Compiles foo.kt with test-runner, probably together with other testData files (bar.kt, qux.kt, ...).
+     *           // Then executes FooKt.one() and FooKt.two() test functions one after another in one shot.
+     *           // If either of test functions fails, the whole "testFoo()" JUnit test is marked as failed.
+     *       }
+     *   }
+     */
+    fun getSingleTestRun(testDataFile: File): TestRun = withTestExecutable(testDataFile) { testCase, executable ->
+        val runParameters = getRunParameters(testCase, testFunction = null)
+        TestRun(displayName = testDataFile.nameWithoutExtension, executable, runParameters, testCase.origin)
+    }
+
+    /**
+     * Produces at least one [TestRun] per testData file.
+     *
+     * If testData file contains multiple functions annotated with [kotlin.test.Test], then a separate [TestRun] will be produced
+     * for each such function.
+     *
+     * This allows to have a better granularity in tests. So that every individual test method inside testData file will be considered
+     * as an individual JUnit test, and will be presented as a separate row in JUnit test report.
+     *
+     * Example:
+     *   //+++ testData file (foo.kt): +++//
+     *   @kotlin.test.Test
+     *   fun one() { /* ... */ }
+     *
+     *   @kotlin.test.Test
+     *   fun two() { /* ... */ }
+     *
+     *   //+++ generated JUnit test suite: +++//
+     *   public class MyTestSuiteGenerated {
+     *       @org.junit.jupiter.api.TestFactory
+     *       @org.jetbrains.kotlin.test.TestMetadata("foo.kt")
+     *       public Collection<org.junit.jupiter.api.DynamicNode> testFoo() {
+     *           // Compiles foo.kt with test-runner, probably together with other testData files (bar.kt, qux.kt, ...).
+     *           // Then produces two instances of DynamicTest for FooKt.one() and FooKt.two() functions.
+     *           // Each DynamicTest is executed as a separate JUnit test.
+     *           // So if FooKt.one() fails and FooKt.two() succeeds, then "testFoo.one" JUnit test will be presented as failed
+     *           // in the test report, and "testFoo.two" will be presented as passed.
+     *       }
+     *   }
+     */
+    fun getTestRuns(testDataFile: File): TreeNode<TestRun> = withTestExecutable(testDataFile) { testCase, executable ->
+        fun createTestRun(testRunName: String, testFunction: TestFunction?): TestRun {
+            val runParameters = getRunParameters(testCase, testFunction)
+            return TestRun(testRunName, executable, runParameters, testCase.origin)
+        }
+
+        when (testCase.kind) {
+            TestKind.STANDALONE_NO_TR -> {
+                val testRunName = testCase.extras<NoTestRunnerExtras>().entryPoint.substringAfterLast('.')
+                val testRun = createTestRun(testRunName, testFunction = null)
+                TreeNode.oneLevel(testRun)
+            }
+            TestKind.REGULAR, TestKind.STANDALONE -> {
+                val testFunctions = testCase.extras<WithTestRunnerExtras>().testFunctions
+                testFunctions.buildTree(TestFunction::packageName) { testFunction -> createTestRun(testFunction.functionName, testFunction) }
+            }
+        }
+    }
+
+    private fun <T> withTestExecutable(testDataFile: File, action: (TestCase, TestExecutable) -> T): T {
        environment.assertNotDisposed()

        val testDataDir = testDataFile.parentFile
@@ -42,13 +127,15 @@ internal class TestRunProvider(
        val testCompilation = when (testCase.kind) {
            TestKind.STANDALONE, TestKind.STANDALONE_NO_TR -> {
                // Create a separate compilation for each standalone test case.
-                cachedCompilations.computeIfAbsent(TestCompilationCacheKey.Standalone(testDataFile)) {
+                val cacheKey = TestCompilationCacheKey.Standalone(testDataFile)
+                cachedCompilations.computeIfAbsent(cacheKey) {
                    compilationFactory.testCasesToExecutable(listOf(testCase))
                }
            }
            TestKind.REGULAR -> {
                // Group regular test cases by compiler arguments.
-                cachedCompilations.computeIfAbsent(TestCompilationCacheKey.Grouped(testDataDir, testCase.freeCompilerArgs)) {
+                val cacheKey = TestCompilationCacheKey.Grouped(testDataDir, testCase.freeCompilerArgs)
+                cachedCompilations.computeIfAbsent(cacheKey) {
                    val testCases = testCaseGroup.getRegularOnlyByCompilerArgs(testCase.freeCompilerArgs)
                    assertTrue(testCases.isNotEmpty())
                    compilationFactory.testCasesToExecutable(testCases)
@@ -59,23 +146,30 @@ internal class TestRunProvider(
        val (executableFile, loggedCompilerCall) = testCompilation.result.assertSuccess() // <-- Compilation happens here.
        val executable = TestExecutable(executableFile, loggedCompilerCall)

-        val runParameters = when (testCase.kind) {
-            TestKind.STANDALONE_NO_TR -> listOfNotNull(
-                testCase.extras!!.inputDataFile?.let(TestRunParameter::WithInputData),
-                testCase.expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
-            )
+        return action(testCase, executable)
+    }
+
+    private fun getRunParameters(testCase: TestCase, testFunction: TestFunction?): List<TestRunParameter> = with(testCase) {
+        when (kind) {
+            TestKind.STANDALONE_NO_TR -> {
+                assertTrue(testFunction == null)
+
+                listOfNotNull(
+                    extras<NoTestRunnerExtras>().inputDataFile?.let(TestRunParameter::WithInputData),
+                    expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
+                )
+            }
            TestKind.STANDALONE -> listOfNotNull(
                TestRunParameter.WithGTestLogger,
-                testCase.expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
+                testFunction?.let(TestRunParameter::WithFunctionFilter),
+                expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
            )
            TestKind.REGULAR -> listOfNotNull(
                TestRunParameter.WithGTestLogger,
-                TestRunParameter.WithPackageFilter(testCase.nominalPackageName),
-                testCase.expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
+                testFunction?.let(TestRunParameter::WithFunctionFilter) ?: TestRunParameter.WithPackageFilter(nominalPackageName),
+                expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData)
            )
        }
-
-        return TestRun(executable, runParameters, testCase.origin)
    }

    // Currently, only local test runner is supported.
@@ -18,6 +18,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.konan.blackboxtest.support.*
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.*
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
@@ -588,7 +589,7 @@ private class ExtTestDataFile(
            origin = TestOrigin.SingleTestDataFile(testDataFile),
            nominalPackageName = settings.nominalPackageName,
            expectedOutputDataFile = null,
-            extras = null
+            extras = WithTestRunnerExtras.EMPTY
        )
        testCase.initialize(sharedModules::get)

@@ -6,6 +6,8 @@
 package org.jetbrains.kotlin.konan.blackboxtest.support.group

 import org.jetbrains.kotlin.konan.blackboxtest.support.*
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.NoTestRunnerExtras
+import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.DEFAULT_FILE_NAME
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.ThreadSafeFactory
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.computeGeneratedSourcesDir
@@ -185,11 +187,24 @@ internal class StandardTestCaseGroupProvider(private val environment: TestEnviro
        val expectedOutputDataFile = parseOutputDataFile(baseDir = testDataFile.parentFile, registeredDirectives, location)
        val testKind = parseTestKind(registeredDirectives, location)

-        if (testKind == TestKind.REGULAR) {
-            // Fix package declarations to avoid unintended conflicts between symbols with the same name in different test cases.
-            testModules.values.forEach { testModule ->
-                testModule.files.forEach { testFile -> fixPackageDeclaration(testFile, nominalPackageName, testDataFile) }
+        val extras = when (testKind) {
+            TestKind.REGULAR, TestKind.STANDALONE -> {
+                // Fix package declarations to avoid unintended conflicts between symbols with the same name in different test cases.
+                val needToFixPackageNames = testKind == TestKind.REGULAR
+
+                val testFunctions = checkExistingPackageNamesAndCollectTestFunctions(
+                    testModules = testModules.values,
+                    basePackageName = nominalPackageName,
+                    fixPackageNames = needToFixPackageNames,
+                    testDataFile = testDataFile
+                )
+
+                WithTestRunnerExtras(testFunctions)
            }
+            TestKind.STANDALONE_NO_TR -> NoTestRunnerExtras(
+                entryPoint = parseEntryPoint(registeredDirectives, location),
+                inputDataFile = parseInputDataFile(baseDir = testDataFile.parentFile, registeredDirectives, location)
+            )
        }

        val testCase = TestCase(
@@ -199,76 +214,130 @@ internal class StandardTestCaseGroupProvider(private val environment: TestEnviro
            origin = TestOrigin.SingleTestDataFile(testDataFile),
            nominalPackageName = nominalPackageName,
            expectedOutputDataFile = expectedOutputDataFile,
-            extras = if (testKind == TestKind.STANDALONE_NO_TR) {
-                TestCase.StandaloneNoTestRunnerExtras(
-                    entryPoint = parseEntryPoint(registeredDirectives, location),
-                    inputDataFile = parseInputDataFile(baseDir = testDataFile.parentFile, registeredDirectives, location)
-                )
-            } else
-                null
+            extras = extras
        )
        testCase.initialize(findSharedModule = null)

        return testCase
    }

-    private fun CharSequence.hasAnythingButComments(): Boolean {
-        var result = false
-        runForFirstMeaningfulStatement { _, _ -> result = true }
-        return result
-    }
-
-    private fun fixPackageDeclaration(
-        testFile: TestFile<TestModule.Exclusive>,
-        packageName: PackageFQN,
-        testDataFile: File
-    ) = testFile.update { text ->
-        var existingPackageDeclarationLine: String? = null
-        var existingPackageDeclarationLineNumber: Int? = null
-
-        text.runForFirstMeaningfulStatement { lineNumber, line ->
-            // First meaningful line.
-            val trimmedLine = line.trim()
-            if (trimmedLine.startsWith("package ")) {
-                existingPackageDeclarationLine = trimmedLine
-                existingPackageDeclarationLineNumber = lineNumber
-            }
+    companion object {
+        private fun CharSequence.hasAnythingButComments(): Boolean {
+            return dropNonMeaningfulLines().firstOrNull() != null
        }

-        if (existingPackageDeclarationLine != null) {
-            val existingPackageName = existingPackageDeclarationLine!!.substringAfter("package ").trimStart()
-            assertTrue(
-                existingPackageName == packageName
-                        || (existingPackageName.length > packageName.length
-                        && existingPackageName.startsWith(packageName)
-                        && existingPackageName[packageName.length] == '.')
-            ) {
-                val location = Location(testDataFile, existingPackageDeclarationLineNumber)
-                """
-                    $location: Invalid package name declaration found: $existingPackageDeclarationLine
-                    Expected: package $packageName
-                """.trimIndent()
-            }
-            text
-        } else
-            "package $packageName $text"
-    }
+        private fun checkExistingPackageNamesAndCollectTestFunctions(
+            testModules: Collection<TestModule.Exclusive>,
+            basePackageName: PackageFQN,
+            fixPackageNames: Boolean,
+            testDataFile: File
+        ): List<TestFunction> {
+            val testFunctions = mutableListOf<TestFunction>()

-    private inline fun CharSequence.runForFirstMeaningfulStatement(action: (lineNumber: Int, line: String) -> Unit) {
-        var inMultilineComment = false
+            testModules.forEach { testModule ->
+                testModule.files.forEach { testFile ->
+                    val meaningfulLines = testFile.text.dropNonMeaningfulLines()

-        for ((lineNumber, line) in lines().withIndex()) {
-            val trimmedLine = line.trim()
-            when {
-                inMultilineComment -> inMultilineComment = !trimmedLine.endsWith("*/")
-                trimmedLine.startsWith("/*") -> inMultilineComment = true
-                trimmedLine.isMeaningfulLine() -> {
-                    action(lineNumber, line)
-                    break
+                    // Retrieve the package name if it is declared inside the test file.
+                    val existingPackageName = meaningfulLines.firstOrNull()?.let { (lineNumber, firstMeaningfulLine) ->
+                        getExistingPackageName(lineNumber, firstMeaningfulLine, basePackageName, testDataFile)
+                    }
+
+                    // Compute the package name to be added (if it should be added, of course).
+                    val packageNameToAdd = if (fixPackageNames && existingPackageName == null) basePackageName else null
+
+                    // Collect test functions.
+                    val effectivePackageName = existingPackageName ?: packageNameToAdd ?: ""
+                    collectTestFunctionsNames(meaningfulLines, testDataFile) { functionName ->
+                        testFunctions += TestFunction(effectivePackageName, functionName)
+                    }
+
+                    // Add package declaration (if necessary).
+                    if (packageNameToAdd != null) {
+                        testFile.update { text -> "package $packageNameToAdd $text" }
+                    }
                }
            }
-        }
-    }

-    private fun String.isMeaningfulLine() = isNotEmpty() && !startsWith("//") && !startsWith("@file:")
+            return testFunctions
+        }
+
+        private fun getExistingPackageName(lineNumber: Int, line: String, basePackageName: PackageFQN, testDataFile: File): PackageFQN? {
+            val existingPackageName = line
+                .substringAfter("package ", missingDelimiterValue = "")
+                .trimStart()
+                .takeIf(String::isNotEmpty)
+                ?: return null
+
+            assertTrue(
+                existingPackageName == basePackageName
+                        || (existingPackageName.length > basePackageName.length
+                        && existingPackageName.startsWith(basePackageName)
+                        && existingPackageName[basePackageName.length] == '.')
+            ) {
+                """
+                   ${Location(testDataFile, lineNumber)}: Invalid package name declaration found: $line
+                    Expected: package $basePackageName
+                """.trimIndent()
+            }
+
+            return existingPackageName
+        }
+
+        private inline fun collectTestFunctionsNames(
+            meaningfulLines: Sequence<Pair<Int, String>>,
+            testDataFile: File,
+            crossinline consumeFunctionName: (FunctionName) -> Unit
+        ) {
+            var expectingTestFunction = false
+
+            val extractFunctionName: (lineNumber: Int, line: String) -> Unit = { lineNumber, line ->
+                val match = FUNCTION_REGEX.matchEntire(line)
+                    ?: fail { "${Location(testDataFile, lineNumber)}: Can't parse name of test function: $line" }
+
+                val functionName = match.groupValues[1]
+
+                consumeFunctionName(functionName)
+            }
+
+            val processPossiblyAnnotatedLine: (lineNumber: Int, line: String) -> Unit = { lineNumber, line ->
+                if (line.startsWith('@'))
+                    for (testAnnotation in TEST_ANNOTATIONS) {
+                        if (line.startsWith(testAnnotation)) {
+                            val remainder = line.substringAfter(testAnnotation).trimStart()
+                            if (remainder.isNotEmpty()) extractFunctionName(lineNumber, remainder) else expectingTestFunction = true
+                            break
+                        }
+                    }
+            }
+
+            meaningfulLines.forEach { (lineNumber, line) ->
+                if (expectingTestFunction) {
+                    extractFunctionName(lineNumber, line)
+                    expectingTestFunction = false
+                } else
+                    processPossiblyAnnotatedLine(lineNumber, line)
+            }
+        }
+
+        private fun CharSequence.dropNonMeaningfulLines(): Sequence<Pair<Int, String>> {
+            var inMultilineComment = false
+
+            return lineSequence()
+                .mapIndexed { lineNumber, line -> lineNumber to line.trim() }
+                .dropWhile { (_, trimmedLine) ->
+                    when {
+                        inMultilineComment -> inMultilineComment = !trimmedLine.endsWith("*/")
+                        trimmedLine.startsWith("/*") -> inMultilineComment = true
+                        trimmedLine.isMeaningfulLine() -> return@dropWhile false
+                    }
+                    return@dropWhile true
+                }
+        }
+
+        private fun String.isMeaningfulLine() = isNotEmpty() && !startsWith("//") && !startsWith("@file:")
+
+        private val TEST_ANNOTATIONS = listOf("@Test", "@kotlin.test.Test")
+        private val FUNCTION_REGEX = Regex("^\\s*fun\\s+(\\p{javaJavaIdentifierPart}+).*$")
+    }
 }
@@ -65,3 +65,9 @@ internal fun Set<PackageFQN>.findCommonPackageName(): PackageFQN? = when (size)
        }
    }.takeIf { it.isNotEmpty() }?.joinToString(".")
 }
+
+internal fun joinPackageNames(a: PackageFQN, b: PackageFQN): PackageFQN = when {
+    a.isEmpty() -> b
+    b.isEmpty() -> a
+    else -> "$a.$b"
+}
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.konan.blackboxtest.support.util
+
+import org.jetbrains.kotlin.konan.blackboxtest.support.PackageFQN
+
+internal interface TreeNode<T> {
+    val packageSegment: String
+    val items: Collection<T>
+    val children: Collection<TreeNode<T>>
+
+    companion object {
+        fun <T> oneLevel(vararg items: T) = oneLevel(listOf(*items))
+
+        fun <T> oneLevel(items: Iterable<T>) = object : TreeNode<T> {
+            override val packageSegment get() = ""
+            override val items = items.toList()
+            override val children get() = emptyList<TreeNode<T>>()
+        }
+    }
+}
+
+internal fun <T, R> Collection<T>.buildTree(extractPackageName: (T) -> PackageFQN, transform: (T) -> R): TreeNode<R> {
+    val groupedItems: Map<PackageFQN, List<R>> = groupBy(extractPackageName).mapValues { (_, items) -> items.map(transform) }
+
+    // Fast pass.
+    when (groupedItems.size) {
+        0 -> return TreeNode.oneLevel()
+        1 -> return TreeNode.oneLevel(groupedItems.values.first())
+    }
+
+    // Long pass.
+    val root = TreeBuilder<R>("")
+
+    // Populate the tree.
+    groupedItems.forEach { (packageFQN, items) ->
+        var node = root
+        packageFQN.split('.').forEach { packageSegment ->
+            node = node.childrenMap.computeIfAbsent(packageSegment) { TreeBuilder(packageSegment) }
+        }
+        node.items += items
+    }
+
+    // Skip meaningless nodes starting from the root. Compress the resulting tree.
+    return root.skipMeaninglessNodes().apply { compress() }
+}
+
+private class TreeBuilder<T>(override var packageSegment: String) : TreeNode<T> {
+    override val items = mutableListOf<T>()
+    val childrenMap = hashMapOf<String, TreeBuilder<T>>()
+    override val children: Collection<TreeBuilder<T>> get() = childrenMap.values
+}
+
+private tailrec fun <T> TreeBuilder<T>.skipMeaninglessNodes(): TreeBuilder<T> =
+    if (items.isNotEmpty() || childrenMap.size != 1)
+        this
+    else
+        childrenMap.values.first().skipMeaninglessNodes()
+
+private fun <T> TreeBuilder<T>.compress() {
+    while (items.isEmpty() && childrenMap.size == 1) {
+        val childNode = childrenMap.values.first()
+
+        items += childNode.items
+
+        childrenMap.clear()
+        childrenMap += childNode.childrenMap
+
+        packageSegment = joinPackageNames(packageSegment, childNode.packageSegment)
+    }
+
+    childrenMap.values.forEach { it.compress() }
+}