From 179d324444a9d533b0477b03048c98462a4102b6 Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Wed, 1 Dec 2021 14:33:22 +0300 Subject: [PATCH] [Native][tests] Extract test function names from test executables --- .../konan/blackboxtest/support/LoggedData.kt | 38 +++-- .../konan/blackboxtest/support/TestCase.kt | 6 +- .../blackboxtest/support/TestCompiler.kt | 2 +- .../blackboxtest/support/TestRunProvider.kt | 62 ++++--- .../support/group/ExtTestCaseGroupProvider.kt | 2 +- .../group/StandardTestCaseGroupProvider.kt | 159 ++++-------------- .../support/runner/AbstractRunner.kt | 10 +- .../runner/LocalTestFunctionExtractor.kt | 42 +++++ .../support/runner/LocalTestRunner.kt | 2 +- .../konan/blackboxtest/support/util/GTest.kt | 69 ++++++++ .../blackboxtest/support/util/PackageNames.kt | 5 + .../support/util/SourceDiscovery.kt | 40 +++++ 12 files changed, 259 insertions(+), 178 deletions(-) create mode 100644 native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestFunctionExtractor.kt create mode 100644 native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/SourceDiscovery.kt diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/LoggedData.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/LoggedData.kt index b60b1f3ac0e..bd948f85f73 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/LoggedData.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/LoggedData.kt @@ -23,8 +23,8 @@ internal abstract class LoggedData { protected abstract fun computeText(): String final override fun toString() = text - fun withErrorMessageHeader(errorMessageHeader: String): String = buildString { - appendLine(errorMessageHeader) + fun withErrorMessage(errorMessage: String): String = buildString { + appendLine(errorMessage) appendLine() appendLine(this@LoggedData) } @@ -90,27 +90,31 @@ internal abstract class LoggedData { class TestRunParameters( private val compilerCall: CompilerCall, - private val testCaseId: TestCaseId, + private val testCaseId: TestCaseId?, private val runArgs: Iterable, - private val runParameters: List + private val runParameters: List? ) : LoggedData() { override fun computeText() = buildString { - if (testCaseId is TestCaseId.TestDataFile) { - appendLine("TEST DATA FILE:") - appendLine(testCaseId.file) - } else { - appendLine("TEST CASE ID:") - appendLine(testCaseId) + when { + testCaseId is TestCaseId.TestDataFile -> { + appendLine("TEST DATA FILE:") + appendLine(testCaseId.file) + appendLine() + } + testCaseId != null -> { + appendLine("TEST CASE ID:") + appendLine(testCaseId) + appendLine() + } } - appendLine() appendArguments("TEST RUN ARGUMENTS:", runArgs) appendLine() - runParameters.get { + runParameters?.get { appendLine("INPUT DATA FILE:") appendLine(inputDataFile) appendLine() } - runParameters.get { + runParameters?.get { appendLine("EXPECTED OUTPUT DATA FILE:") appendLine(expectedOutputDataFile) appendLine() @@ -198,13 +202,13 @@ internal abstract class LoggedData { appendLine("- Exit code: ${runResult.exitCode ?: ""}") appendDuration(runResult.duration) appendLine() - appendLine("========== BEGIN: TEST STDOUT ==========") + appendLine("========== BEGIN: STDOUT ==========") if (runResult.output.stdOut.isNotEmpty()) appendLine(runResult.output.stdOut.trimEnd()) - appendLine("========== END: TEST STDOUT ==========") + appendLine("========== END: STDOUT ==========") appendLine() - appendLine("========== BEGIN: TEST STDERR ==========") + appendLine("========== BEGIN: STDERR ==========") if (runResult.output.stdErr.isNotEmpty()) appendLine(runResult.output.stdErr.trimEnd()) - appendLine("========== END: TEST STDERR ==========") + appendLine("========== END: STDERR ==========") return this } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt index 409e31ae907..e8cfd4efbcb 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt @@ -180,11 +180,7 @@ internal class TestCase( ) { sealed interface Extras class NoTestRunnerExtras(val entryPoint: String, val inputDataFile: File?) : Extras - class WithTestRunnerExtras(val testFunctions: Collection) : Extras { - companion object { - val EMPTY = WithTestRunnerExtras(emptyList()) - } - } + object WithTestRunnerExtras : Extras init { when (kind) { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCompiler.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCompiler.kt index 919a98e3089..e986d98c62c 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCompiler.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCompiler.kt @@ -181,7 +181,7 @@ internal sealed interface TestCompilationResult { is DependencyFailures -> fail { describeDependencyFailures() } } - private fun Failure.describeFailure() = loggedData.withErrorMessageHeader( + private fun Failure.describeFailure() = loggedData.withErrorMessage( when (this@describeFailure) { is CompilerFailure -> "Compilation failed." is UnexpectedFailure -> "Compilation failed with unexpected exception." diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestRunProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestRunProvider.kt index 644489507c2..9cb97636dba 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestRunProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestRunProvider.kt @@ -9,16 +9,17 @@ package org.jetbrains.kotlin.konan.blackboxtest.support import com.intellij.openapi.util.Disposer import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.NoTestRunnerExtras -import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras import org.jetbrains.kotlin.konan.blackboxtest.support.TestCompilationResult.Companion.assertSuccess import org.jetbrains.kotlin.konan.blackboxtest.support.group.TestCaseGroupProvider import org.jetbrains.kotlin.konan.blackboxtest.support.runner.AbstractRunner +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.LocalTestFunctionExtractor import org.jetbrains.kotlin.konan.blackboxtest.support.runner.LocalTestRunner import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GlobalSettings import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings import org.jetbrains.kotlin.konan.blackboxtest.support.util.ThreadSafeCache import org.jetbrains.kotlin.konan.blackboxtest.support.util.TreeNode import org.jetbrains.kotlin.konan.blackboxtest.support.util.buildTree +import org.jetbrains.kotlin.konan.blackboxtest.support.util.isSameOrSubpackageOf import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail import org.junit.jupiter.api.Assumptions.assumeTrue @@ -31,6 +32,7 @@ internal class TestRunProvider( ) : ExtensionContext.Store.CloseableResource { private val compilationFactory = TestCompilationFactory(settings) private val cachedCompilations = ThreadSafeCache() + private val cachedTestFunctions = ThreadSafeCache>() fun setProcessors(testDataFile: File, sourceTransformers: List<(String) -> String>) { testCaseGroupProvider.setPreprocessors(testDataFile, sourceTransformers) @@ -61,7 +63,7 @@ internal class TestRunProvider( * } * } */ - fun getSingleTestRun(testCaseId: TestCaseId): TestRun = withTestExecutable(testCaseId) { testCase, executable -> + fun getSingleTestRun(testCaseId: TestCaseId): TestRun = withTestExecutable(testCaseId) { testCase, executable, _ -> val runParameters = getRunParameters(testCase, testFunction = null) TestRun(displayName = /* Unimportant. Used only in dynamic tests. */ "", executable, runParameters, testCase.id) } @@ -96,7 +98,7 @@ internal class TestRunProvider( * } * } */ - fun getTestRuns(testCaseId: TestCaseId): TreeNode = withTestExecutable(testCaseId) { testCase, executable -> + fun getTestRuns(testCaseId: TestCaseId): TreeNode = withTestExecutable(testCaseId) { testCase, executable, cacheKey -> fun createTestRun(testRunName: String, testFunction: TestFunction?): TestRun { val runParameters = getRunParameters(testCase, testFunction) return TestRun(testRunName, executable, runParameters, testCase.id) @@ -109,7 +111,10 @@ internal class TestRunProvider( TreeNode.oneLevel(testRun) } TestKind.REGULAR, TestKind.STANDALONE -> { - val testFunctions = testCase.extras().testFunctions + val testFunctions = cachedTestFunctions.computeIfAbsent(cacheKey) { + extractTestFunctions(executable) + }.filterIrrelevant(testCase) + testFunctions.buildTree(TestFunction::packageName) { testFunction -> createTestRun(testFunction.functionName, testFunction) } @@ -117,7 +122,7 @@ internal class TestRunProvider( } } - private fun withTestExecutable(testCaseId: TestCaseId, action: (TestCase, TestExecutable) -> T): T { + private fun withTestExecutable(testCaseId: TestCaseId, action: (TestCase, TestExecutable, TestCompilationCacheKey) -> T): T { settings.assertNotDisposed() val testCaseGroup = testCaseGroupProvider.getTestCaseGroup(testCaseId.testCaseGroupId) ?: fail { "No test case for $testCaseId" } @@ -125,36 +130,37 @@ internal class TestRunProvider( val testCase = testCaseGroup.getByName(testCaseId) ?: fail { "No test case for $testCaseId" } - val testCompilation = when (testCase.kind) { + val (testCompilation, cacheKey) = when (testCase.kind) { TestKind.STANDALONE, TestKind.STANDALONE_NO_TR -> { // Create a separate compilation for each standalone test case. val cacheKey = TestCompilationCacheKey.Standalone(testCaseId) - cachedCompilations.computeIfAbsent(cacheKey) { + val testCompilation = cachedCompilations.computeIfAbsent(cacheKey) { compilationFactory.testCasesToExecutable(listOf(testCase)) } + testCompilation to cacheKey } TestKind.REGULAR -> { // Group regular test cases by compiler arguments. val cacheKey = TestCompilationCacheKey.Grouped(testCaseId.testCaseGroupId, testCase.freeCompilerArgs) - cachedCompilations.computeIfAbsent(cacheKey) { + val testCompilation = cachedCompilations.computeIfAbsent(cacheKey) { val testCases = testCaseGroup.getRegularOnlyByCompilerArgs(testCase.freeCompilerArgs) assertTrue(testCases.isNotEmpty()) compilationFactory.testCasesToExecutable(testCases) } + testCompilation to cacheKey } } val (executableFile, loggedCompilerCall) = testCompilation.result.assertSuccess() // <-- Compilation happens here. val executable = TestExecutable(executableFile, loggedCompilerCall) - return action(testCase, executable) + return action(testCase, executable, cacheKey) } private fun getRunParameters(testCase: TestCase, testFunction: TestFunction?): List = with(testCase) { when (kind) { TestKind.STANDALONE_NO_TR -> { assertTrue(testFunction == null) - listOfNotNull( extras().inputDataFile?.let(TestRunParameter::WithInputData), expectedOutputDataFile?.let(TestRunParameter::WithExpectedOutputData) @@ -175,16 +181,32 @@ internal class TestRunProvider( // Currently, only local test runner is supported. fun createRunner(testRun: TestRun): AbstractRunner<*> = with(settings.get()) { - when (val target = target) { - hostTarget -> LocalTestRunner(testRun, executionTimeout) - else -> fail { - """ - Running at non-host target is not supported yet. - Compilation target: $target - Host target: $hostTarget - """.trimIndent() - } - } + if (target == hostTarget) + LocalTestRunner(testRun, executionTimeout) + else + runningAtNonHostTarget() + } + + // Currently, only local test function extractor is supported. + private fun extractTestFunctions(executable: TestExecutable): Collection = with(settings.get()) { + if (target == hostTarget) + LocalTestFunctionExtractor(executable, executionTimeout).run() + else + runningAtNonHostTarget() + } + + private fun Collection.filterIrrelevant(testCase: TestCase) = + if (testCase.kind == TestKind.REGULAR) + filter { testFunction -> testFunction.packageName.isSameOrSubpackageOf(testCase.nominalPackageName) } + else + this + + private fun GlobalSettings.runningAtNonHostTarget(): Nothing = fail { + """ + Running at non-host target is not supported yet. + Compilation target: $target + Host target: $hostTarget + """.trimIndent() } override fun close() { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt index cb4d464fc2a..d12d37f5943 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/ExtTestCaseGroupProvider.kt @@ -594,7 +594,7 @@ private class ExtTestDataFile( freeCompilerArgs = assembleFreeCompilerArgs(), nominalPackageName = testDataFileSettings.nominalPackageName, expectedOutputDataFile = null, - extras = WithTestRunnerExtras.EMPTY + extras = WithTestRunnerExtras ) testCase.initialize(sharedModules::get) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt index 4bad4148395..bd58ef1b409 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/group/StandardTestCaseGroupProvider.kt @@ -11,6 +11,7 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerEx import org.jetbrains.kotlin.konan.blackboxtest.support.settings.GeneratedSources import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestRoots +import org.jetbrains.kotlin.konan.blackboxtest.support.util.* import org.jetbrains.kotlin.konan.blackboxtest.support.util.DEFAULT_FILE_NAME import org.jetbrains.kotlin.konan.blackboxtest.support.util.ThreadSafeFactory import org.jetbrains.kotlin.konan.blackboxtest.support.util.computeGeneratedSourcesDir @@ -194,24 +195,9 @@ internal class StandardTestCaseGroupProvider(private val settings: Settings) : T val expectedOutputDataFile = parseOutputDataFile(baseDir = testDataFile.parentFile, registeredDirectives, location) val testKind = parseTestKind(registeredDirectives, location) - val extras = when (testKind) { - TestKind.REGULAR, TestKind.STANDALONE -> { - // Fix package declarations to avoid unintended conflicts between symbols with the same name in different test cases. - val needToFixPackageNames = testKind == TestKind.REGULAR - - val testFunctions = checkExistingPackageNamesAndCollectTestFunctions( - testModules = testModules.values, - basePackageName = nominalPackageName, - fixPackageNames = needToFixPackageNames, - testDataFile = testDataFile - ) - - WithTestRunnerExtras(testFunctions) - } - TestKind.STANDALONE_NO_TR -> NoTestRunnerExtras( - entryPoint = parseEntryPoint(registeredDirectives, location), - inputDataFile = parseInputDataFile(baseDir = testDataFile.parentFile, registeredDirectives, location) - ) + if (testKind == TestKind.REGULAR) { + // Fix package declarations to avoid unintended conflicts between symbols with the same name in different test cases. + fixPackageNames(testModules.values, nominalPackageName, testDataFile) } val testCase = TestCase( @@ -221,7 +207,13 @@ internal class StandardTestCaseGroupProvider(private val settings: Settings) : T freeCompilerArgs = freeCompilerArgs, nominalPackageName = nominalPackageName, expectedOutputDataFile = expectedOutputDataFile, - extras = extras + extras = if (testKind == TestKind.STANDALONE_NO_TR) + NoTestRunnerExtras( + entryPoint = parseEntryPoint(registeredDirectives, location), + inputDataFile = parseInputDataFile(baseDir = testDataFile.parentFile, registeredDirectives, location) + ) + else + WithTestRunnerExtras ) testCase.initialize(findSharedModule = null) @@ -229,122 +221,33 @@ internal class StandardTestCaseGroupProvider(private val settings: Settings) : T } companion object { - private fun CharSequence.hasAnythingButComments(): Boolean { - return dropNonMeaningfulLines().firstOrNull() != null - } - - private fun checkExistingPackageNamesAndCollectTestFunctions( - testModules: Collection, - basePackageName: PackageFQN, - fixPackageNames: Boolean, - testDataFile: File - ): List { - val testFunctions = mutableListOf() - + private fun fixPackageNames(testModules: Collection, basePackageName: PackageFQN, testDataFile: File) { testModules.forEach { testModule -> testModule.files.forEach { testFile -> - val meaningfulLines = testFile.text.dropNonMeaningfulLines() + val firstMeaningfulLine = testFile.text.dropNonMeaningfulLines().firstOrNull() // Retrieve the package name if it is declared inside the test file. - val existingPackageName = meaningfulLines.firstOrNull()?.let { (lineNumber, firstMeaningfulLine) -> - getExistingPackageName(lineNumber, firstMeaningfulLine, basePackageName, testDataFile) - } - - // Compute the package name to be added (if it should be added, of course). - val packageNameToAdd = if (fixPackageNames && existingPackageName == null) basePackageName else null - - // Collect test functions. - val effectivePackageName = existingPackageName ?: packageNameToAdd ?: "" - collectTestFunctionsNames(meaningfulLines, testDataFile) { functionName -> - testFunctions += TestFunction(effectivePackageName, functionName) - } - - // Add package declaration (if necessary). - if (packageNameToAdd != null) { - testFile.update { text -> "package $packageNameToAdd $text" } - } - } - } - - return testFunctions - } - - private fun getExistingPackageName(lineNumber: Int, line: String, basePackageName: PackageFQN, testDataFile: File): PackageFQN? { - val existingPackageName = line - .substringAfter("package ", missingDelimiterValue = "") - .trimStart() - .takeIf(String::isNotEmpty) - ?: return null - - assertTrue( - existingPackageName == basePackageName - || (existingPackageName.length > basePackageName.length - && existingPackageName.startsWith(basePackageName) - && existingPackageName[basePackageName.length] == '.') - ) { - """ - ${Location(testDataFile, lineNumber)}: Invalid package name declaration found: $line - Expected: package $basePackageName - """.trimIndent() - } - - return existingPackageName - } - - private inline fun collectTestFunctionsNames( - meaningfulLines: Sequence>, - testDataFile: File, - crossinline consumeFunctionName: (FunctionName) -> Unit - ) { - var expectingTestFunction = false - - val extractFunctionName: (lineNumber: Int, line: String) -> Unit = { lineNumber, line -> - val match = FUNCTION_REGEX.matchEntire(line) - ?: fail { "${Location(testDataFile, lineNumber)}: Can't parse name of test function: $line" } - - val functionName = match.groupValues[1] - - consumeFunctionName(functionName) - } - - val processPossiblyAnnotatedLine: (lineNumber: Int, line: String) -> Unit = { lineNumber, line -> - if (line.startsWith('@')) - for (testAnnotation in TEST_ANNOTATIONS) { - if (line.startsWith(testAnnotation)) { - val remainder = line.substringAfter(testAnnotation).trimStart() - if (remainder.isNotEmpty()) extractFunctionName(lineNumber, remainder) else expectingTestFunction = true - break + val existingPackageName = firstMeaningfulLine?.getExistingPackageName() + if (existingPackageName != null) { + // Validate it. + assertTrue( + existingPackageName == basePackageName + || (existingPackageName.length > basePackageName.length + && existingPackageName.startsWith(basePackageName) + && existingPackageName[basePackageName.length] == '.') + ) { + val location = Location(testDataFile, firstMeaningfulLine.number) + """ + $location: Invalid package name declaration found: $firstMeaningfulLine + Expected: package $basePackageName + """.trimIndent() } + } else { + // Add package declaration. + testFile.update { text -> "package $basePackageName $text" } } - } - - meaningfulLines.forEach { (lineNumber, line) -> - if (expectingTestFunction) { - extractFunctionName(lineNumber, line) - expectingTestFunction = false - } else - processPossiblyAnnotatedLine(lineNumber, line) - } - } - - private fun CharSequence.dropNonMeaningfulLines(): Sequence> { - var inMultilineComment = false - - return lineSequence() - .mapIndexed { lineNumber, line -> lineNumber to line.trim() } - .dropWhile { (_, trimmedLine) -> - when { - inMultilineComment -> inMultilineComment = !trimmedLine.endsWith("*/") - trimmedLine.startsWith("/*") -> inMultilineComment = true - trimmedLine.isMeaningfulLine() -> return@dropWhile false - } - return@dropWhile true } + } } - - private fun String.isMeaningfulLine() = isNotEmpty() && !startsWith("//") && !startsWith("@file:") - - private val TEST_ANNOTATIONS = listOf("@Test", "@kotlin.test.Test") - private val FUNCTION_REGEX = Regex("^\\s*fun\\s+(\\p{javaJavaIdentifierPart}+).*$") } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt index ab45f239b8d..17be1b3fd44 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt @@ -22,7 +22,7 @@ internal abstract class AbstractRunner { val resultHandler = when (val runResult = run.run()) { is RunResult.TimeoutExceeded -> fail { LoggedData.TestRunTimeoutExceeded(getLoggedParameters(), runResult) - .withErrorMessageHeader("Timeout exceeded during test execution.") + .withErrorMessage("Timeout exceeded during test execution.") } is RunResult.Completed -> buildResultHandler(runResult) } @@ -45,12 +45,12 @@ internal abstract class AbstractRunner { abstract fun getLoggedRun(): LoggedData abstract fun handle(): R - protected inline fun verifyExpectation(expected: T, actual: T, crossinline errorMessageHeader: () -> String) { - assertEquals(expected, actual) { getLoggedRun().withErrorMessageHeader(errorMessageHeader()) } + protected inline fun verifyExpectation(expected: T, actual: T, crossinline errorMessage: () -> String) { + assertEquals(expected, actual) { getLoggedRun().withErrorMessage(errorMessage()) } } - protected inline fun verifyExpectation(shouldBeTrue: Boolean, crossinline errorMessageHeader: () -> String) { - assertTrue(shouldBeTrue) { getLoggedRun().withErrorMessageHeader(errorMessageHeader()) } + protected inline fun verifyExpectation(shouldBeTrue: Boolean, crossinline errorMessage: () -> String) { + assertTrue(shouldBeTrue) { getLoggedRun().withErrorMessage(errorMessage()) } } } } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestFunctionExtractor.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestFunctionExtractor.kt new file mode 100644 index 00000000000..3bf18877b77 --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestFunctionExtractor.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.blackboxtest.support.runner + +import org.jetbrains.kotlin.konan.blackboxtest.support.LoggedData +import org.jetbrains.kotlin.konan.blackboxtest.support.TestExecutable +import org.jetbrains.kotlin.konan.blackboxtest.support.TestFunction +import org.jetbrains.kotlin.konan.blackboxtest.support.util.parseGTestListing +import org.jetbrains.kotlin.test.services.JUnit5Assertions +import kotlin.time.Duration + +internal class LocalTestFunctionExtractor( + override val executable: TestExecutable, + executionTimeout: Duration +) : AbstractLocalProcessRunner>(executionTimeout) { + override val visibleProcessName get() = "Test function extractor" + override val programArgs = listOf(executable.executableFile.path, "--ktest_list_tests") + + override fun getLoggedParameters() = LoggedData.TestRunParameters( + compilerCall = executable.loggedCompilerCall, + testCaseId = null, + runArgs = programArgs, + runParameters = null + ) + + override fun buildResultHandler(runResult: RunResult.Completed) = ResultHandler(runResult) + + override fun handleUnexpectedFailure(t: Throwable) = JUnit5Assertions.fail { + LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t) + .withErrorMessage("Test function extraction failed with unexpected exception.") + } + + inner class ResultHandler( + runResult: RunResult.Completed + ) : AbstractLocalProcessRunner>.ResultHandler(runResult) { + override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult) + override fun doHandle() = parseGTestListing(runResult.output.stdOut) + } +} diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt index 5a5fd482532..f9acb4abc54 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt @@ -41,7 +41,7 @@ internal class LocalTestRunner( override fun handleUnexpectedFailure(t: Throwable) = fail { LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t) - .withErrorMessageHeader("Test execution failed with unexpected exception.") + .withErrorMessage("Test execution failed with unexpected exception.") } inner class ResultHandler(runResult: RunResult.Completed) : AbstractLocalProcessRunner.ResultHandler(runResult) { diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/GTest.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/GTest.kt index 16b90d300e3..2f2c54665f2 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/GTest.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/GTest.kt @@ -5,6 +5,11 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.util +import org.jetbrains.kotlin.konan.blackboxtest.support.PackageFQN +import org.jetbrains.kotlin.konan.blackboxtest.support.TestFunction +import org.jetbrains.kotlin.konan.blackboxtest.support.util.GTestListingParseState.* +import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail + internal typealias TestName = String internal typealias TestStatus = String @@ -68,5 +73,69 @@ internal fun parseGTestReport(stdOut: String): GTestReport { return GTestReport(testStatuses, cleanStdOut.toString()) } +/** + * Extracts [TestFunction]s from GTest listing. + * + * Example: + * sample.test.SampleTestKt. + * one + * two + * + * yields TestFunction(sample.test, one) and TestFunction(sample.test, two). + */ +internal fun parseGTestListing(rawGTestListing: String): Collection = buildList { + var state: GTestListingParseState = Begin + + rawGTestListing.lineSequence().forEachIndexed { index, line -> + fun parseError(message: String): Nothing = fail { + buildString { + appendLine("$message at line #$index: \"$line\"") + appendLine() + appendLine("Full listing:") + appendLine(rawGTestListing) + } + } + + state = when { + line.isBlank() -> when (state) { + is NewTest, is End -> End + else -> parseError("Unexpected empty line") + } + line[0].isWhitespace() -> when (val s = state) { + is HasPackageName -> { + this += TestFunction(s.packageName, line.trim()) + NewTest(s.packageName) + } + else -> parseError("Test name encountered before test suite name") + } + else -> when (state) { + is Begin, is NewTest -> { + val packageParts = line.trimEnd().removeSuffix(".").split('.') + if (packageParts.isEmpty()) parseError("Malformed test suite name") + + // Drop the last part because it is related to class name (or file-class name). + // TODO: How to handle nested classes? + val packageName = packageParts.dropLast(1).joinToString(".") + + NewTestSuite(packageName) + } + else -> parseError("Unexpected test suite name") + } + } + } +} + private val RUN_LINE_REGEX = Regex("""^\[\s+RUN\s+]\s+.*""") private val STATUS_LINE_REGEX = Regex("""^\[\s+([A-Z]+)\s+]\s+(\S+)\s+.*""") + +private sealed interface GTestListingParseState { + object Begin : GTestListingParseState + object End : GTestListingParseState + + interface HasPackageName : GTestListingParseState { + val packageName: PackageFQN + } + + class NewTestSuite(override val packageName: PackageFQN) : HasPackageName + class NewTest(override val packageName: PackageFQN) : HasPackageName +} diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/PackageNames.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/PackageNames.kt index 3ac1e19dd52..38a605ab9d4 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/PackageNames.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/PackageNames.kt @@ -74,3 +74,8 @@ internal fun joinPackageNames(a: PackageFQN, b: PackageFQN): PackageFQN = when { internal fun String.prependPackageName(packageName: PackageFQN): String = if (packageName.isEmpty()) this else "$packageName.$this" + +internal fun PackageFQN.isSameOrSubpackageOf(parentPackageName: PackageFQN): Boolean = + parentPackageName.isEmpty() + || this == parentPackageName + || (length > parentPackageName.length && startsWith(parentPackageName) && this[parentPackageName.length] == '.') diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/SourceDiscovery.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/SourceDiscovery.kt new file mode 100644 index 00000000000..23635b30602 --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/util/SourceDiscovery.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.blackboxtest.support.util + +import org.jetbrains.kotlin.konan.blackboxtest.support.PackageFQN + +internal typealias SourceText = CharSequence +internal typealias SourceLine = String + +internal data class NumberedSourceLine(val number: Int, val text: SourceLine) : CharSequence by text { + override fun toString() = text +} + +internal fun SourceText.hasAnythingButComments(): Boolean = + dropNonMeaningfulLines().firstOrNull() != null + +internal fun NumberedSourceLine.getExistingPackageName(): PackageFQN? = + text.substringAfter("package ", missingDelimiterValue = "") + .trimStart() + .takeIf(String::isNotEmpty) + +internal fun SourceText.dropNonMeaningfulLines(): Sequence { + var inMultilineComment = false + + return lineSequence() + .mapIndexed { lineNumber, line -> NumberedSourceLine(lineNumber, line.trim()) } + .dropWhile { (_, trimmedLine) -> + when { + inMultilineComment -> inMultilineComment = !trimmedLine.endsWith("*/") + trimmedLine.startsWith("/*") -> inMultilineComment = true + trimmedLine.isMeaningfulLine() -> return@dropWhile false + } + return@dropWhile true + } +} + +private fun SourceLine.isMeaningfulLine() = isNotEmpty() && !startsWith("//") && !startsWith("@file:")