From 26ffd66a54b73e0e1971d29e57179e45d0ee12a8 Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Fri, 11 Feb 2022 19:14:20 +0300 Subject: [PATCH] [Native][tests] Tests to check dumped test listing ^KT-50316 --- .../testListing/bar/common-tests.kt | 21 +++ .../testListing/bar/util-tests.kt | 9 + .../testListing/expected-test-listing.dump | 5 + .../testListing/foo/common-tests.kt | 21 +++ .../infrastructure/testListing/foo/util.kt | 3 + .../InfrastructureDumpedTestListingTest.kt | 171 ++++++++++++++++++ .../support/compilation/TestCompilation.kt | 1 + .../compilation/TestCompilationArtifact.kt | 1 + 8 files changed, 232 insertions(+) create mode 100644 native/native.tests/testData/infrastructure/testListing/bar/common-tests.kt create mode 100644 native/native.tests/testData/infrastructure/testListing/bar/util-tests.kt create mode 100644 native/native.tests/testData/infrastructure/testListing/expected-test-listing.dump create mode 100644 native/native.tests/testData/infrastructure/testListing/foo/common-tests.kt create mode 100644 native/native.tests/testData/infrastructure/testListing/foo/util.kt create mode 100644 native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureDumpedTestListingTest.kt diff --git a/native/native.tests/testData/infrastructure/testListing/bar/common-tests.kt b/native/native.tests/testData/infrastructure/testListing/bar/common-tests.kt new file mode 100644 index 00000000000..92246611c8f --- /dev/null +++ b/native/native.tests/testData/infrastructure/testListing/bar/common-tests.kt @@ -0,0 +1,21 @@ +package infrastructure.testListing.bar + +import kotlin.test.* + +@Test +fun topLevel() = Unit + +class Outer { + @Test + fun outer() = Unit + + class Nested { + @Test + fun nested() = Unit + } +} + +object O { + @Test + fun testObject() = Unit +} diff --git a/native/native.tests/testData/infrastructure/testListing/bar/util-tests.kt b/native/native.tests/testData/infrastructure/testListing/bar/util-tests.kt new file mode 100644 index 00000000000..b501590919a --- /dev/null +++ b/native/native.tests/testData/infrastructure/testListing/bar/util-tests.kt @@ -0,0 +1,9 @@ +package infrastructure.testListing.bar + +import kotlin.test.* +import infrastructure.testListing.foo.fortyTwo + +@Test +fun testFortyTwo() { + assertEquals(42, fortyTwo()) +} diff --git a/native/native.tests/testData/infrastructure/testListing/expected-test-listing.dump b/native/native.tests/testData/infrastructure/testListing/expected-test-listing.dump new file mode 100644 index 00000000000..b66bbdd6578 --- /dev/null +++ b/native/native.tests/testData/infrastructure/testListing/expected-test-listing.dump @@ -0,0 +1,5 @@ +infrastructure/testListing/bar/Common_testsKt:topLevel +infrastructure/testListing/bar/O:testObject +infrastructure/testListing/bar/Outer.Nested:nested +infrastructure/testListing/bar/Outer:outer +infrastructure/testListing/bar/Util_testsKt:testFortyTwo diff --git a/native/native.tests/testData/infrastructure/testListing/foo/common-tests.kt b/native/native.tests/testData/infrastructure/testListing/foo/common-tests.kt new file mode 100644 index 00000000000..19341fdb2d4 --- /dev/null +++ b/native/native.tests/testData/infrastructure/testListing/foo/common-tests.kt @@ -0,0 +1,21 @@ +package infrastructure.testListing.foo + +import kotlin.test.* + +@Test +fun topLevel() = Unit + +class Outer { + @Test + fun outer() = Unit + + class Nested { + @Test + fun nested() = Unit + } +} + +object O { + @Test + fun testObject() = Unit +} diff --git a/native/native.tests/testData/infrastructure/testListing/foo/util.kt b/native/native.tests/testData/infrastructure/testListing/foo/util.kt new file mode 100644 index 00000000000..9bc986dc1a9 --- /dev/null +++ b/native/native.tests/testData/infrastructure/testListing/foo/util.kt @@ -0,0 +1,3 @@ +package infrastructure.testListing.foo + +fun fortyTwo() = 42 diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureDumpedTestListingTest.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureDumpedTestListingTest.kt new file mode 100644 index 00000000000..766d3fab88b --- /dev/null +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/InfrastructureDumpedTestListingTest.kt @@ -0,0 +1,171 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.konan.blackboxtest + +import com.intellij.testFramework.TestDataFile +import com.intellij.testFramework.TestDataPath +import org.jetbrains.kotlin.konan.blackboxtest.InfrastructureDumpedTestListingTest.Companion.TEST_SUITE_PATH +import org.jetbrains.kotlin.konan.blackboxtest.support.* +import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras +import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.ExecutableCompilation +import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.ExistingDependency +import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.LibraryCompilation +import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationArtifact.Executable +import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationArtifact.KLIB +import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationDependency +import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationDependencyType.IncludedLibrary +import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationDependencyType.Library +import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Companion.assertSuccess +import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationResult.Success +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestExecutable +import org.jetbrains.kotlin.konan.blackboxtest.support.runner.TestRunners +import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeTargets +import org.jetbrains.kotlin.konan.blackboxtest.support.settings.SimpleTestDirectories +import org.jetbrains.kotlin.konan.blackboxtest.support.util.DumpedTestListing +import org.jetbrains.kotlin.konan.blackboxtest.support.util.LAUNCHER_MODULE_NAME +import org.jetbrains.kotlin.test.TestMetadata +import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertEquals +import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.Test +import java.io.File + +@Tag("infrastructure") +@TestMetadata(TEST_SUITE_PATH) +@TestDataPath("\$PROJECT_ROOT") +class InfrastructureDumpedTestListingTest : AbstractNativeSimpleTest() { + @Test + @TestMetadata(TEST_CASE_NAME) + fun testListingCompiledFromSources() { + doTest(TEST_CASE_PATH, fromSources = true) + } + + @Test + @TestMetadata(TEST_CASE_NAME) + fun testListingCompiledFromIncludedLibrary() { + doTest(TEST_CASE_PATH, fromSources = false) + } + + @Suppress("SameParameterValue") + private fun doTest(@TestDataFile testDataPath: String, fromSources: Boolean) { + val rootDir = File(testDataPath) + + val fooTestCase: TestCase = generateTestCaseWithSingleModule(rootDir.resolve("foo")) + val fooCompilationResult: Success = fooTestCase.compileToLibrary() + val fooLibrary: KLIB = fooCompilationResult.resultingArtifact + + val barTestCase: TestCase = generateTestCaseWithSingleModule(rootDir.resolve("bar")) + + val executableTestCase: TestCase + val executableCompilationResult: Success + + if (fromSources) { + executableTestCase = barTestCase + executableCompilationResult = barTestCase.compileToExecutable(fooLibrary.asLibraryDependency()) + } else { + val barCompilationResult: Success = barTestCase.compileToLibrary(fooLibrary.asLibraryDependency()) + val barLibrary: KLIB = barCompilationResult.resultingArtifact + + executableTestCase = generateTestCaseWithSingleModule(moduleDir = null) // No sources. + executableCompilationResult = executableTestCase.compileToExecutable( + fooLibrary.asLibraryDependency(), + barLibrary.asIncludedLibraryDependency() + ) + } + + val executable: Executable = executableCompilationResult.resultingArtifact + + // check that the test listing dumped during the compilation matches our expectations: + val testDumpFile = executable.testDumpFile + assertDumpFilesEqual(expected = rootDir.resolve("expected-test-listing.dump"), actual = testDumpFile) + + // parse test listing that was dumped to a file during compilation: + val dumpedTestListing = DumpedTestListing.parse(testDumpFile.readText()).toSet() + assertTrue(dumpedTestListing.isNotEmpty()) + + // parse test listing obtained from executable file with the help of --ktest_list_tests flag: + val testExecutable = TestExecutable(executable.executableFile, executableCompilationResult.loggedData) + val extractedTestListing = TestRunners.extractTestNames(testExecutable, testRunSettings).toSet() + + assertEquals(extractedTestListing, dumpedTestListing) + + runExecutableAndVerify(executableTestCase, testExecutable) // <-- run executable and verify + } + + private fun generateTestCaseWithSingleModule(moduleDir: File?): TestCase { + val moduleName: String = moduleDir?.name ?: LAUNCHER_MODULE_NAME + val module = TestModule.Exclusive(moduleName, emptySet(), emptySet()) + + moduleDir?.walkTopDown() + ?.filter { it.isFile && it.extension == "kt" } + ?.forEach { file -> module.files += TestFile.createCommitted(file, module) } + + return TestCase( + id = TestCaseId.Named(moduleName), + kind = TestKind.STANDALONE, + modules = setOf(module), + freeCompilerArgs = TestCompilerArgs.EMPTY, + nominalPackageName = PackageName.EMPTY, + expectedOutputDataFile = null, + extras = DEFAULT_EXTRAS + ).apply { + initialize(null) + } + } + + private fun TestCase.compileToLibrary(vararg dependencies: TestCompilationDependency<*>): Success { + val compilation = LibraryCompilation( + settings = testRunSettings, + freeCompilerArgs = freeCompilerArgs, + sourceModules = modules, + dependencies = dependencies.toList(), + expectedArtifact = toLibraryArtifact() + ) + return compilation.result.assertSuccess() + } + + private fun TestCase.compileToExecutable(vararg dependencies: TestCompilationDependency<*>): Success { + val compilation = ExecutableCompilation( + settings = testRunSettings, + freeCompilerArgs = freeCompilerArgs, + sourceModules = modules, + extras = DEFAULT_EXTRAS, + dependencies = dependencies.toList(), + expectedArtifact = toExecutableArtifact() + ) + return compilation.result.assertSuccess() + } + + private val buildDir: File get() = testRunSettings.get().testBuildDir + + private fun TestCase.toLibraryArtifact() = KLIB(buildDir.resolve(modules.first().name + ".klib")) + private fun toExecutableArtifact() = + Executable(buildDir.resolve("app." + testRunSettings.get().testTarget.family.exeSuffix)) + + private fun KLIB.asLibraryDependency() = ExistingDependency(this, Library) + private fun KLIB.asIncludedLibraryDependency() = ExistingDependency(this, IncludedLibrary) + + private fun assertDumpFilesEqual(expected: File, actual: File) { + val expectedDumpFileContents = expected.readText().trimEnd() + val actualDumpFileContents = actual.readText().trimEnd() + + assertEquals(expectedDumpFileContents, actualDumpFileContents) { + """ + Test dump file contents mismatch. + Expected: ${expected.absolutePath} + Actual: ${actual.absolutePath} + """.trimIndent() + } + } + + companion object { + const val TEST_SUITE_PATH = "native/native.tests/testData/infrastructure" + const val TEST_CASE_NAME = "testListing" + const val TEST_CASE_PATH = "$TEST_SUITE_PATH/$TEST_CASE_NAME" + + private val DEFAULT_EXTRAS = WithTestRunnerExtras(TestRunnerType.DEFAULT) + } +} diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/compilation/TestCompilation.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/compilation/TestCompilation.kt index 690b755f869..2b6f6cc96b5 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/compilation/TestCompilation.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/compilation/TestCompilation.kt @@ -246,6 +246,7 @@ internal class ExecutableCompilation( TestRunnerType.NO_EXIT -> "-generate-no-exit-test-runner" } add(testRunnerArg) + add("-Xdump-tests-to=${expectedArtifact.testDumpFile}") } } super.applySpecificArgs(argsBuilder) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/compilation/TestCompilationArtifact.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/compilation/TestCompilationArtifact.kt index da253709982..491387b6c3e 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/compilation/TestCompilationArtifact.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/compilation/TestCompilationArtifact.kt @@ -22,5 +22,6 @@ internal sealed interface TestCompilationArtifact { data class Executable(val executableFile: File) : TestCompilationArtifact { val path: String get() = executableFile.path override val logFile: File get() = executableFile.resolveSibling("${executableFile.name}.log") + val testDumpFile: File get() = executableFile.resolveSibling("${executableFile.name}.dump") } }