From c15d9ee81a375f94277b74256afca8dd5d5987a9 Mon Sep 17 00:00:00 2001 From: Pavel Punegov Date: Thu, 19 Dec 2019 18:51:38 +0300 Subject: [PATCH] Test module support for external tests. Code cleanup for external tests runner class. --- backend.native/tests/build.gradle | 34 +- backend.native/tests/helpers.kt | 6 - backend.native/tests/testUtils.kt | 26 -- build-tools/build.gradle.kts | 5 + .../org/jetbrains/kotlin/KonanTest.groovy | 301 ++++++------------ .../org/jetbrains/kotlin/KotlinNativeTest.kt | 9 +- .../org/jetbrains/kotlin/TestDirectives.kt | 110 +++++-- .../main/kotlin/org/jetbrains/kotlin/Utils.kt | 7 +- 8 files changed, 212 insertions(+), 286 deletions(-) delete mode 100644 backend.native/tests/helpers.kt delete mode 100644 backend.native/tests/testUtils.kt diff --git a/backend.native/tests/build.gradle b/backend.native/tests/build.gradle index afa32deb940..d003ee6d82b 100644 --- a/backend.native/tests/build.gradle +++ b/backend.native/tests/build.gradle @@ -106,12 +106,12 @@ final CacheTesting cacheTesting = CacheTestingKt.configureCacheTesting(project) if (cacheTesting != null) { // Note: can't do this in [CacheTesting.configure] since task classes aren't accessible there. - tasks.withType(KonanCompileNativeBinary.class) { + tasks.withType(KonanCompileNativeBinary.class).configureEach { dependsOn cacheTesting.buildCacheTask extraOpts cacheTesting.compilerArgs } - tasks.withType(RunExternalTestGroup.class) { + tasks.withType(RunExternalTestGroup.class).configureEach { dependsOn cacheTesting.buildCacheTask flags = (flags ?: []) + cacheTesting.compilerArgs } @@ -120,11 +120,11 @@ if (cacheTesting != null) { // Enable two-stage test compilation if the test_two_stage property is set. ext.twoStageEnabled = project.hasProperty("test_two_stage") -tasks.withType(KonanCompileNativeBinary.class) { +tasks.withType(KonanCompileNativeBinary.class).configureEach { enableTwoStageCompilation = twoStageEnabled } -tasks.withType(RunExternalTestGroup.class) { +tasks.withType(RunExternalTestGroup.class).configureEach { enableTwoStageCompilation = twoStageEnabled } @@ -290,22 +290,20 @@ boolean isExcluded(String dir) { return result } -def createTestTasks(File testRoot, Class taskType, Closure taskConfiguration) { +def createTestTasks(File testRoot, Class taskType) { testRoot.eachDirRecurse { // Skip build directory and directories without *.kt files. if (it.name == "build" || it.listFiles().count { it.name.endsWith(".kt") && it.isFile() } == 0) { return } - def taskDirectory = project.relativePath(it) + def taskDirectory = project.relativePath(it).substring("build/".length()) if (!isExcluded(taskDirectory)) { def taskName = taskDirectory.replaceAll("[-/\\\\]", '_') - - def task = project.task(taskName, type: taskType) { - groupDirectory = taskDirectory - finalizedBy resultsTask + project.tasks.register(taskName, taskType) { + it.groupDirectory = taskDirectory + it.finalizedBy resultsTask } - taskConfiguration(task) } else { println("$taskDirectory is excluded") } @@ -404,7 +402,7 @@ task run_external () { } // Create tasks for external tests. - createTestTasks(externalTestsDir, RunExternalTestGroup) { } + createTestTasks(externalTestsDir, RunExternalTestGroup) // Set up dependencies. dependsOn(tasksOf(RunExternalTestGroup)) @@ -4399,7 +4397,7 @@ task buildKonanTests { t -> // These tests should not be built into the TestRunner's test executable def excludeList = [ "codegen/inline/returnLocalClassFromBlock.kt" ] project.tasks - .matching { it instanceof KonanStandaloneTest } + .withType(KonanStandaloneTest.class) .each { // add library and source files if (it instanceof KonanLinkTest) { @@ -4426,7 +4424,8 @@ task buildKonanTests { t -> // Local tests build into a single binary should depend on this task project.tasks - .matching { !(it instanceof KonanStandaloneTest) && it instanceof KonanLocalTest } + .withType(KonanLocalTest.class) + .matching { !(it instanceof KonanStandaloneTest) } .forEach { it.dependsOn(t) } } @@ -4478,12 +4477,13 @@ dependencies { // Configure build for iOS device targets. if (target.family == Family.IOS && (target.architecture == ARM32 || target.architecture == ARM64)) { project.tasks - .matching { it instanceof KonanTestExecutable } + .withType(KonanTestExecutable.class) .forEach { ExecutorServiceKt.configureXcodeBuild((KonanTestExecutable) it) } - // Exclude tasks that can not be run on device + // Exclude tasks that cannot be run on device project.tasks + .withType(KonanTest.class) .matching { (it instanceof KonanLocalTest && ((KonanLocalTest) it).testData != null) || // Filter out tests that depend on libs. TODO: copy them too it instanceof KonanInteropTest || @@ -4491,4 +4491,4 @@ if (target.family == Family.IOS && (target.architecture == ARM32 || target.archi it instanceof KonanLinkTest } .forEach { it.enabled = false } -} +} \ No newline at end of file diff --git a/backend.native/tests/helpers.kt b/backend.native/tests/helpers.kt deleted file mode 100644 index 8e03b1964d9..00000000000 --- a/backend.native/tests/helpers.kt +++ /dev/null @@ -1,6 +0,0 @@ -/* - * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -package helpers \ No newline at end of file diff --git a/backend.native/tests/testUtils.kt b/backend.native/tests/testUtils.kt deleted file mode 100644 index 2f8be14f232..00000000000 --- a/backend.native/tests/testUtils.kt +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license - * that can be found in the LICENSE file. - */ - -package kotlin.test - -@Suppress("UNUSED_PARAMETER") -public fun assertTypeEquals(expected: Any?, actual: Any?) { - //TODO: find analogue - //assertEquals(expected?.javaClass, actual?.javaClass) -} - -fun Iterable.assertSorted(isInOrder: (T, T) -> Boolean): Unit { this.iterator().assertSorted(isInOrder) } -fun Iterator.assertSorted(isInOrder: (T, T) -> Boolean) { - if (!hasNext()) return - var index = 0 - var prev = next() - while (hasNext()) { - index += 1 - val next = next() - assertTrue(isInOrder(prev, next), "Not in order at position $index, element[${index-1}]: $prev, element[$index]: $next") - prev = next - } - return -} diff --git a/build-tools/build.gradle.kts b/build-tools/build.gradle.kts index b7515ffd3bf..9aad8bb0dd0 100644 --- a/build-tools/build.gradle.kts +++ b/build-tools/build.gradle.kts @@ -42,9 +42,13 @@ repositories { maven("https://kotlin.bintray.com/kotlinx") } +val kotlinCompilerJar by configurations.creating + dependencies { compileOnly(gradleApi()) + kotlinCompilerJar("org.jetbrains.kotlin:kotlin-compiler:${kotlinVersion}@jar") + implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion") implementation("org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion") implementation("org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion") @@ -92,6 +96,7 @@ val compileGroovy: GroovyCompile by tasks // Add Kotlin classes to a classpath for the Groovy compiler compileGroovy.apply { classpath += project.files(compileKotlin.destinationDir) + classpath += kotlinCompilerJar dependsOn(compileKotlin) } diff --git a/build-tools/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy b/build-tools/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy index 00a164370b4..d7129f4d7a9 100644 --- a/build-tools/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy +++ b/build-tools/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy @@ -20,24 +20,19 @@ import org.gradle.api.tasks.Input import org.gradle.api.tasks.JavaExec import org.gradle.api.tasks.TaskAction import org.gradle.process.ExecResult +import org.jetbrains.kotlin.utils.DFS -import javax.inject.Inject import java.nio.file.Paths import java.util.function.Function +import java.util.function.UnaryOperator import java.util.regex.Pattern +import java.util.stream.Collectors -abstract class OldKonanTest extends JavaExec { - public boolean inDevelopersRun = false - - public String source +class RunExternalTestGroup extends JavaExec { def platformManager = project.rootProject.platformManager def target = platformManager.targetManager(project.testTarget).target - def dist = project.rootProject.file(project.findProperty("org.jetbrains.kotlin.native.home") ?: - project.findProperty("konan.home") ?: "dist") - def dependenciesDir = project.rootProject.dependenciesDir - def konancDriver = project.isWindows() ? "konanc.bat" : "konanc" - def konanc = new File("${dist.canonicalPath}/bin/$konancDriver").absolutePath - def outputSourceSetName = "testOutputLocal" + def dist = UtilsKt.getKotlinNativeDist(project) + def enableKonanAssertions = true String outputDirectory = null String goldValue = null @@ -52,44 +47,22 @@ abstract class OldKonanTest extends JavaExec { boolean multiRuns = false List> multiArguments = null - boolean enabled = true boolean expectedFail = false - boolean run = true boolean compilerMessages = false - void setDisabled(boolean value) { - this.enabled = !value - } - - void setExpectedFail(boolean value) { - this.expectedFail = value - } - - // Uses directory defined in $outputSourceSetName source set. - // If such source set doesn't exist, uses temporary directory. + // Uses directory defined in $outputSourceSetName source set void createOutputDirectory() { if (outputDirectory != null) { return } - def outputSourceSet = project.findProperty(getOutputSourceSetName()) - if (outputSourceSet != null) { - outputDirectory = outputSourceSet.absolutePath + "/$name" - project.file(outputDirectory).mkdirs() - } else { - outputDirectory = getTemporaryDir().absolutePath - } + def outputSourceSet = UtilsKt.getTestOutputExternal(project) + outputDirectory = Paths.get(outputSourceSet, name).toString() + project.file(outputDirectory).mkdirs() } - OldKonanTest() { + RunExternalTestGroup() { // We don't build the compiler if a custom dist path is specified. - if (!project.ext.useCustomDist) { - dependsOn(project.rootProject.tasks['dist']) - if (project.testTarget) { - // if a test_target property is set then tests should depend on a crossDist - // otherwise runtime components would not be build for a target - dependsOn(project.rootProject.tasks["${target}CrossDist"]) - } - } + UtilsKt.dependsOnDist(this) } @Override @@ -99,8 +72,6 @@ abstract class OldKonanTest extends JavaExec { // configuration part at runCompiler. } - abstract void compileTest(List filesToCompile, String exe) - protected void runCompiler(List filesToCompile, String output, List moreArgs) { def log = new ByteArrayOutputStream() try { @@ -144,91 +115,20 @@ abstract class OldKonanTest extends JavaExec { } } - protected void runCompiler(String source, String output, List moreArgs) { - runCompiler([source], output, moreArgs) - } - - String buildExePath() { - def exeName = project.file(source).name.replace(".kt", "") - return "$outputDirectory/$exeName" - } - - protected String removeDiagnostics(String str) { - return str.replaceAll(~/(.*?)/) { all, text -> text } - } - - protected List registerKtFile(List sourceFiles, String newFilePath, String newFileContent) { - createFile(newFilePath, newFileContent) - if (newFilePath.endsWith(".kt")) { - sourceFiles.add(newFilePath) - } - return sourceFiles - } - - // TODO refactor - List buildCompileList() { - def result = [] - def filePattern = ~/(?m)\/\/\s*FILE:\s*(.*)$/ - def srcFile = project.file(source) - def srcText = removeDiagnostics(srcFile.text) - def matcher = filePattern.matcher(srcText) - - if (srcText.contains('// WITH_COROUTINES')) { - def coroutineHelpersFileName = "$outputDirectory/helpers.kt" - createFile(coroutineHelpersFileName, CoroutineTestUtilKt.createTextForHelpers(true)) - result.add(coroutineHelpersFileName) - } - - if (!matcher.find()) { - // There is only one file in the input - def filePath = "$outputDirectory/${srcFile.name}" - registerKtFile(result, filePath, srcText) - } else { - // There are several files - def processedChars = 0 - while (true) { - def filePath = "$outputDirectory/${matcher.group(1)}" - def start = processedChars - def nextFileExists = matcher.find() - def end = nextFileExists ? matcher.start() : srcText.length() - def fileText = srcText.substring(start, end) - processedChars = end - registerKtFile(result, filePath, fileText) - if (!nextFileExists) break - } - } - return result - } - - void createFile(String file, String text) { - Paths.get(file).with { - getParent().toFile().with { - if (!exists()) { mkdirs() } - } - write(text) - } - } + // FIXME: output directory here changes and hence this is not a property + String executablePath() { return "$outputDirectory/program.tr" } OutputStream out - @TaskAction - void executeTest() { + void runExecutable() { if (!enabled) { println "Test is disabled: $name" return } - - createOutputDirectory() - def program = buildExePath() + def program = executablePath() def suffix = target.family.exeSuffix def exe = "$program.$suffix" - compileTest(buildCompileList(), program) - - if (!run) { - println "to be executed manually: $exe" - return - } println "execution: $exe" def compilerMessagesText = compilerMessages ? project.file("${program}.compilation.log").getText('UTF-8') : "" @@ -290,13 +190,6 @@ abstract class OldKonanTest extends JavaExec { if (!exitCodeMismatch && !goldValueMismatch && this.expectedFail) println("Unexpected pass") } -} - -class RunExternalTestGroup extends OldKonanTest { - /** - * overrides [KonanTest::inDevelopersRun] used in [:backend.native:tests:sanity] - */ - public def inDevelopersRun = false /** * If true, the test executable will be built in two stages: @@ -306,22 +199,15 @@ class RunExternalTestGroup extends OldKonanTest { @Input public def enableTwoStageCompilation = false + @Input def groupDirectory = "." - def outputSourceSetName = "testOutputExternal" + String filter = project.findProperty("filter") + def testGroupReporter = new KonanTestGroupReportEnvironment(project) - RunExternalTestGroup() { - } - - @Override - List buildCompileList() { - // Already build by the previous step - return null - } - - void parseLanguageFlags() { - def text = project.file(source).text + void parseLanguageFlags(String src) { + def text = project.buildDir.toPath().resolve(src).text def languageSettings = findLinesWithPrefixesRemoved(text, "// !LANGUAGE: ") if (languageSettings.size() != 0) { languageSettings.forEach { line -> @@ -352,7 +238,7 @@ class RunExternalTestGroup extends OldKonanTest { return result.join(System.lineSeparator()) } - String insertInTextAfter(String text, String insert, String after) { + static String insertInTextAfter(String text, String insert, String after) { def begin = text.indexOf(after) if (begin != -1) { def end = text.indexOf("\n", begin) @@ -363,7 +249,7 @@ class RunExternalTestGroup extends OldKonanTest { return text } - List createTestFiles() { + List createTestFiles(String src) { def identifier = /[a-zA-Z_][a-zA-Z0-9_]/ def fullQualified = /[a-zA-Z_][a-zA-Z0-9_.]/ def importRegex = /(?m)^\s*import\s+/ @@ -372,18 +258,20 @@ class RunExternalTestGroup extends OldKonanTest { def boxPattern = ~/(?m)fun\s+box\s*\(\s*\)/ def classPattern = ~/.*(class|object|enum|interface)\s+(${identifier}*).*/ - def sourceName = "_" + normalize(project.file(source).name) + def sourceName = "_" + normalize(project.buildDir.toPath().resolve(src).toFile().name) def packages = new LinkedHashSet() def imports = [] def classes = [] + TestModule mainModule = null - def result = super.buildCompileList() - for (String filePath : result) { - def text = project.file(filePath).text + def testFiles = TestDirectivesKt.buildCompileList(project.file("build/$src").toPath(), "$outputDirectory/$src") + for (TestFile testFile: testFiles) { + def text = testFile.text + def filePath = testFile.path if (text.contains('COROUTINES_PACKAGE')) { text = text.replace('COROUTINES_PACKAGE', 'kotlin.coroutines') } - def pkg = null + def pkg if (text =~ packagePattern) { pkg = (text =~ packagePattern)[0][1] packages.add(pkg) @@ -395,18 +283,17 @@ class RunExternalTestGroup extends OldKonanTest { } if (text =~ boxPattern) { imports.add("${pkg}.*") + mainModule = testFile.module } // Find mutable objects that should be marked as ThreadLocal - if (filePath != "$outputDirectory/helpers.kt") { + if (filePath != "$outputDirectory/$src/helpers.kt") { text = markMutableObjects(text) } - - createFile(filePath, text) + testFile.text = text } - // TODO: optimize files writes - for (String filePath : result) { - def text = project.file(filePath).text + for (TestFile testFile: testFiles) { + def text = testFile.text // Find if there are any imports in the file def matcher = (text =~ ~/${importRegex}(${fullQualified}*)/) if (matcher) { @@ -444,7 +331,7 @@ class RunExternalTestGroup extends OldKonanTest { text = insertInTextAfter(text, (pkg ? "\n$pkg\n" : "") + "import $sourceName.*\n", "@file:Suppress") } // now replace all package usages in full qualified names - def res = "" // result + def res = "" // filesToCompile def vars = new HashSet() // variables that has the same name as a package text.eachLine { line -> packages.each { pkg -> @@ -452,8 +339,8 @@ class RunExternalTestGroup extends OldKonanTest { if ((line =~ ~/va(l|r) *$pkg *\=/) || (line =~ ~/fun .*\(\n?\s*$pkg:.*/)) { vars.add(pkg) } - if (line.contains("$pkg.") && ! (line =~ packagePattern || line =~ importRegex) - && ! vars.contains(pkg)) { + if (line.contains("$pkg.") && !(line =~ packagePattern || line =~ importRegex) + && !vars.contains(pkg)) { def idx = 0 while ((idx = line.indexOf(pkg, idx)) >= 0) { if (!Character.isJavaIdentifierPart(line.charAt(idx - 1))) { @@ -467,11 +354,12 @@ class RunExternalTestGroup extends OldKonanTest { } res += "$line\n" } - createFile(filePath, res) + testFile.text = res } - createLauncherFile("$outputDirectory/_launcher.kt", imports) - result.add("$outputDirectory/_launcher.kt") - return result + def launcherText = createLauncherFileText(src, imports) + testFiles.add(new TestFile("_launcher.kt", "$outputDirectory/$src/_launcher.kt".toString(), + launcherText, mainModule != null ? mainModule : TestModule.default)) + return testFiles } String normalize(String name) { @@ -483,14 +371,13 @@ class RunExternalTestGroup extends OldKonanTest { /** * There are tests that require non-trivial 'package foo' in test launcher. */ - void createLauncherFile(String file, List imports) { + String createLauncherFileText(String src, List imports) { StringBuilder text = new StringBuilder() - def pack = normalize(project.file(source).name) + def pack = normalize(project.file(src).name) text.append("package _$pack\n") for (v in imports) { - text.append("import ").append(v).append('\n') + text.append("import $v\n") } - text.append( """ import kotlin.test.Test @@ -500,10 +387,9 @@ fun runTest() { @Suppress("UNUSED_VARIABLE") val result = box() if (result != "OK") throw AssertionError("Test failed with: " + result) - print(result) } """ ) - createFile(file, text.toString()) + return text.toString() } List findLinesWithPrefixesRemoved(String text, String prefix) { @@ -522,7 +408,7 @@ fun runTest() { ] boolean isEnabledForNativeBackend(String fileName) { - def text = project.file(fileName).text + def text = project.buildDir.toPath().resolve(fileName).text if (excludeList.contains(fileName.replace(File.separator, "/"))) return false @@ -570,31 +456,11 @@ fun runTest() { } } - @Override - void compileTest(List filesToCompile, String exe) { - // An executable should be already compiled - } - - @Override - String buildExePath() { - def outputDir - def outputSourceSet = project.findProperty(getOutputSourceSetName()) - if (outputSourceSet != null) { - outputDir = outputSourceSet.absolutePath+ "/$name" - } else { - outputDir = getTemporaryDir().absolutePath - } - return "$outputDir/program.tr" - } - @TaskAction - @Override void executeTest() { createOutputDirectory() - def outputRootDirectory = outputDirectory - // Form the test list. - List ktFiles = project.file(groupDirectory) + List ktFiles = project.buildDir.toPath().resolve(groupDirectory).toFile() .listFiles({ it.isFile() && it.name.endsWith(".kt") } as FileFilter) @@ -608,29 +474,53 @@ fun runTest() { testGroupReporter.suite(name) { suite -> // Build tests in the group flags = (flags ?: []) + "-tr" - def compileList = [] + List compileList = [] ktFiles.each { - source = project.relativePath(it) - if (isEnabledForNativeBackend(source)) { + def src = project.buildDir.relativePath(it) + if (isEnabledForNativeBackend(src)) { // Create separate output directory for each test in the group. - outputDirectory = outputRootDirectory + "/${it.name}" - project.file(outputDirectory).mkdirs() - parseLanguageFlags() - compileList.addAll(createTestFiles()) + project.file("$outputDirectory/${it.name}").mkdirs() + parseLanguageFlags(src) + compileList.addAll(createTestFiles(src)) } } - compileList.add(project.file("testUtils.kt").absolutePath) - compileList.add(project.file("helpers.kt").absolutePath) + compileList*.writeTextToFile() try { - def exePath = buildExePath() if (enableTwoStageCompilation) { // Two-stage compilation. - def klibPath = "${exePath}.klib" - runCompiler(compileList, klibPath, flags + ["-p", "library"]) - runCompiler([], exePath, flags + ["-Xinclude=$klibPath"]) + def klibPath = "${executablePath()}.klib" + def files = compileList.stream() + .map { it.path } + .collect(Collectors.toList()) + runCompiler(files, klibPath, flags + ["-p", "library"]) + runCompiler([], executablePath(), flags + ["-Xinclude=$klibPath"]) } else { - // Regular compilation. - runCompiler(compileList, exePath, flags) + // Regular compilation with modules. + Map modules = compileList.stream() + .map { it.module } + .distinct() + .collect(Collectors.toMap({ it.name }, UnaryOperator.identity() )) + + List orderedModules = DFS.topologicalOrder(modules.values()) { module -> + module.dependencies.collect { modules[it] }.findAll { it != null } + } + def libsFlags = [] + orderedModules.reverse().each { module -> + if (!module.isDefaultModule()) { + def klibModulePath = "${executablePath()}.${module.name}.klib" + libsFlags += module.linkDependencies(executablePath()) + runCompiler(compileList.findAll { it.module == module }.collect { it.path }, + klibModulePath, flags + ["-p", "library"] + libsFlags) + } + } + + def compileMain = compileList.findAll { + it.module.isDefaultModule() || it.module == TestModule.support + } + compileMain.forEach { f -> + libsFlags.addAll(f.module.linkDependencies(executablePath())) + } + if (!compileMain.empty) runCompiler(compileMain.collect { it.path }, executablePath(), flags + libsFlags) } } catch (Exception ex) { project.logger.quiet("ERROR: Compilation failed for test suite: $name with exception", ex) @@ -641,20 +531,21 @@ fun runTest() { } // Run the tests. - def currentResult = null - outputDirectory = outputRootDirectory arguments = (arguments ?: []) + "--ktest_logger=SILENT" ktFiles.each { file -> - source = project.relativePath(file) + def src = project.buildDir.relativePath(file) def savedArgs = arguments arguments += "--ktest_filter=_${normalize(file.name)}.*" use(KonanTestSuiteReportKt) { - project.logger.quiet("TEST: $file.name (done: $testGroupReporter.statistics.total/${ktFiles.size()}, passed: $testGroupReporter.statistics.passed, skipped: $testGroupReporter.statistics.skipped)") + project.logger.quiet("TEST: $file.name " + + "(done: $testGroupReporter.statistics.total/${ktFiles.size()}, " + + "passed: $testGroupReporter.statistics.passed, " + + "skipped: $testGroupReporter.statistics.skipped)") } - if (isEnabledForNativeBackend(source)) { + if (isEnabledForNativeBackend(src)) { suite.executeTest(file.name) { - project.logger.quiet(source) - super.executeTest() + project.logger.quiet(src) + runExecutable() } } else { suite.skipTest(file.name) diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/KotlinNativeTest.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/KotlinNativeTest.kt index ef3e296221a..097395d4386 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/KotlinNativeTest.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/KotlinNativeTest.kt @@ -9,6 +9,7 @@ import org.gradle.api.Action import org.gradle.api.DefaultTask import org.gradle.api.Project import org.gradle.api.Task +import org.gradle.api.provider.Provider import org.gradle.api.tasks.Input import org.gradle.api.tasks.Optional import org.gradle.api.tasks.TaskAction @@ -334,7 +335,11 @@ open class KonanStandaloneTest : KonanLocalTest() { var flags: List = listOf() get() = if (enableKonanAssertions) field + "-ea" else field - fun getSources() = buildCompileList(outputDirectory) + fun getSources(): Provider> = project.provider { + val sources = buildCompileList(project.file(source).toPath(), outputDirectory) + sources.forEach { it.writeTextToFile() } + sources.map { it.path } + } } /** @@ -363,7 +368,7 @@ open class KonanDriverTest : KonanStandaloneTest() { add("-target") add(project.testTarget.visibleName) } - addAll(getSources()) + addAll(getSources().get()) addAll(flags) addAll(project.globalTestArgs) } diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/TestDirectives.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/TestDirectives.kt index dc0c2bb2861..327a6bc05c7 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/TestDirectives.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/TestDirectives.kt @@ -5,60 +5,114 @@ package org.jetbrains.kotlin +import java.nio.file.Path import java.nio.file.Paths import java.util.regex.Pattern +private const val MODULE_DELIMITER = ",\\s*" +private val FILE_OR_MODULE_PATTERN: Pattern = Pattern.compile("(?://\\s*MODULE:\\s*([^()\\n]+)(?:\\(([^()]+(?:" + + "$MODULE_DELIMITER[^()]+)*)\\))?\\s*(?:\\(([^()]+(?:$MODULE_DELIMITER[^()]+)*)\\))?\\s*)?//\\s*FILE:\\s*(.*)$", + Pattern.MULTILINE) + /** - * Creates files from the given source file that may contain different test directives. + * Creates test files from the given source file that may contain different test directives. * - * @return list of file names to be compiled + * @return list of test files [TestFile] to be compiled */ -fun KonanTest.buildCompileList(outputDirectory: String): List { - val result = mutableListOf() - val srcFile = project.file(source) +fun buildCompileList(source: Path, outputDirectory: String): List { + val result = mutableListOf() + val srcFile = source.toFile() // Remove diagnostic parameters in external tests. val srcText = srcFile.readText().replace(Regex("(.*?)")) { match -> match.groupValues[1] } if (srcText.contains("// WITH_COROUTINES")) { - val coroutineHelpersFileName = "$outputDirectory/helpers.kt" - createFile(coroutineHelpersFileName, createTextForHelpers(true)) - result.add(coroutineHelpersFileName) + result.add(TestFile("helpers.kt", "$outputDirectory/helpers.kt", + createTextForHelpers(true), TestModule.support)) } - val filePattern = Pattern.compile("(?m)// *FILE: *(.*)") - val matcher = filePattern.matcher(srcText) - + val matcher = FILE_OR_MODULE_PATTERN.matcher(srcText) if (!matcher.find()) { // There is only one file in the input - val filePath = "$outputDirectory/${srcFile.name}" - registerKtFile(result, filePath, srcText) + result.add(TestFile(srcFile.name, "$outputDirectory/${srcFile.name}", srcText)) } else { // There are several files var processedChars = 0 - while (true) { - val filePath = "$outputDirectory/${matcher.group(1)}" + var module: TestModule = TestModule.default + var nextFileExists = true + while (nextFileExists) { + var moduleName = matcher.group(1) + val moduleDependencies = matcher.group(2) + val moduleFriends = matcher.group(3) + + if (moduleName != null) { + moduleName = moduleName.trim { it <= ' ' } + module = TestModule("${srcFile.name}.$moduleName", + moduleDependencies.parseModuleList().map { + if (it != "support") "${srcFile.name}.$it" else it + }, + moduleFriends.parseModuleList().map { "${srcFile.name}.$it" }) + } + + val fileName = matcher.group(4) + val filePath = "$outputDirectory/$fileName" val start = processedChars - val nextFileExists = matcher.find() + nextFileExists = matcher.find() val end = if (nextFileExists) matcher.start() else srcText.length val fileText = srcText.substring(start, end) processedChars = end - registerKtFile(result, filePath, fileText) - if (!nextFileExists) break + if (fileName.endsWith(".kt")) { + result.add(TestFile(fileName, filePath, fileText, module)) + } } } return result } -internal fun createFile(file: String, text: String) = Paths.get(file).run { - parent.toFile() - .takeUnless { it.exists() } - ?.mkdirs() - toFile().writeText(text) -} +private fun String?.parseModuleList() = this + ?.split(Pattern.compile(MODULE_DELIMITER), 0) + ?: emptyList() -internal fun registerKtFile(sourceFiles: MutableList, newFilePath: String, newFileContent: String) { - createFile(newFilePath, newFileContent) - if (newFilePath.endsWith(".kt")) { - sourceFiles.add(newFilePath) +/** + * Test module from the test source declared by the [FILE_OR_MODULE_PATTERN]. + * Module should have a [name] and could have [dependencies] on other modules and [friends]. + * + * There are 2 predefined modules: + * - [default] that contains all sources that don't declare a module, + * - [support] for a helper sources like Coroutines support. + */ +data class TestModule( + val name: String, + val dependencies: List, + val friends: List +) { + fun isDefaultModule() = this == default || name.endsWith(".main") + + fun linkDependencies(outputPath: String): List = + dependencies.flatMap { listOf("-l", "$outputPath.$it.klib") } + + companion object { + val default = TestModule("default", emptyList(), emptyList()) + val support = TestModule("support", emptyList(), emptyList()) + } +} + +/** + * Represent a single test file that belongs to the [module]. + */ +data class TestFile(val name: String, + val path: String, + var text: String = "", + val module: TestModule = TestModule.default +) { + /** + * Writes [text] to the file created from the [path]. + */ + fun writeTextToFile() { + Paths.get(path).takeUnless { text.isEmpty() }?.run { + parent.toFile() + .takeUnless { it.exists() } + ?.mkdirs() + toFile().writeText(text) + } } } diff --git a/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt b/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt index baaa039b916..d32c3035a0f 100644 --- a/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt +++ b/build-tools/src/main/kotlin/org/jetbrains/kotlin/Utils.kt @@ -46,6 +46,9 @@ val Project.testOutputStdlib val Project.testOutputFramework get() = (findProperty("testOutputFramework") as File).toString() +val Project.testOutputExternal + get() = (findProperty("testOutputExternal") as File).toString() + val Project.kotlinNativeDist get() = this.rootProject.file(this.findProperty("org.jetbrains.kotlin.native.home") ?: this.findProperty("konan.home") ?: "dist") @@ -254,8 +257,8 @@ fun compileSwift(project: Project, target: KonanTarget, sources: List, o |stdout: $stdOut |stderr: $stdErr """.trimMargin()) - check(exitCode == 0, { "Compilation failed" }) - check(output.toFile().exists(), { "Compiler swiftc hasn't produced an output file: $output" }) + check(exitCode == 0) { "Compilation failed" } + check(output.toFile().exists()) { "Compiler swiftc hasn't produced an output file: $output" } } fun targetSupportsMimallocAllocator(targetName: String) =