From fd44baf2c018bccea3fb469484c68daa9f55dc4b Mon Sep 17 00:00:00 2001 From: Ilya Matveev Date: Fri, 13 Jan 2017 18:47:48 +0700 Subject: [PATCH] backend/tests: Infrastructure for external test running --- .gitignore | 1 + backend.native/tests/build.gradle | 152 ++++++--------- backend.native/tests/external/build.gradle | 0 .../org/jetbrains/kotlin/KonanTest.groovy | 183 ++++++++++++++++++ settings.gradle | 1 + 5 files changed, 246 insertions(+), 91 deletions(-) create mode 100644 backend.native/tests/external/build.gradle create mode 100644 buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy diff --git a/.gitignore b/.gitignore index f29f98b5108..120809d3411 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ kotstd/kotstd.iml *.kt.S *.kt.exe *.log +**/test-result.md # Ignore Gradle GUI config gradle-app.setting diff --git a/backend.native/tests/build.gradle b/backend.native/tests/build.gradle index 00ed3811ce4..676a7272511 100644 --- a/backend.native/tests/build.gradle +++ b/backend.native/tests/build.gradle @@ -1,3 +1,10 @@ +import com.google.common.collect.Lists +import groovy.io.FileType +import org.jetbrains.kotlin.* + +import java.nio.file.* +import java.util.regex.Matcher + configurations { cli_bc } @@ -6,101 +13,38 @@ dependencies { cli_bc project(path: ':backend.native', configuration: 'cli_bc') } +task regenerate_external_tests() { + doLast { + def externalTestsProject = childProjects["external"] + def gradleGenerated = externalTestsProject.file("build.gradle") + def externalTestsDir = externalTestsProject.file("codegen/blackbox") + gradleGenerated.write("import org.jetbrains.kotlin.*\n\n") + gradleGenerated.append("configurations {\n" + + " cli_bc\n" + + "}\n" + + "\n" + + "dependencies {\n" + + " cli_bc project(path: ':backend.native', configuration: 'cli_bc')\n" + + "}\n\n") - -abstract class KonanTest extends DefaultTask { - protected String source - def backendNative = project.project(":backend.native") - def runtimeProject = project.project(":runtime") - def dist = project.parent.parent.file("dist") - def llvmLlc = llvmTool("llc") - def runtimeBc = new File("${dist.canonicalPath}/lib/runtime.bc").absolutePath - def launcherBc = new File("${dist.canonicalPath}/lib/launcher.bc").absolutePath - def startKtBc = new File("${dist.canonicalPath}/lib/start.kt.bc").absolutePath - def stdlibKtBc = new File("${dist.canonicalPath}/lib/stdlib.kt.bc").absolutePath - def mainC = 'main.c' - String goldValue = null - String testData = null - List arguments = null - - boolean enabled = true - - public void setDisabled(boolean value) { - this.enabled = !value - } - - public KonanTest(){ - // TODO: that's a long reach up the project tree. - // May be we should reorganize a little. - dependsOn(project.parent.parent.tasks['dist']) - } - - abstract void compileTest(String source, String exe) - - protected void runCompiler(String source, String output, List moreArgs) { - project.javaexec { - main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt' - classpath = project.configurations.cli_bc - jvmArgs "-ea", - "-Dkonan.home=${dist.canonicalPath}", - "-Djava.library.path=${dist.canonicalPath}/konan/nativelib" - args("-output", output, - source, - *moreArgs, - *project.globalArgs) - } - } - - @TaskAction - void executeTest() { - def sourceKt = project.file(source).absolutePath - def exe = sourceKt.replace(".kt", ".kt.exe") - - compileTest(sourceKt, exe) - println "execution :$exe" - - def out = null - project.exec { - commandLine exe - if (arguments != null) { - args arguments + externalTestsDir.eachDirRecurse { + if (it.name == "build") { + return } - if (testData != null) { - standardInput = new ByteArrayInputStream(testData.bytes) - } - if (goldValue != null) { - out = new ByteArrayOutputStream() - standardOutput = out + + def directoryGradleGenerated = project.file("$it.absolutePath/build-generated.gradle") + directoryGradleGenerated.write("import org.jetbrains.kotlin.*\n\n") + gradleGenerated.append("\napply from: \"${externalTestsProject.relativePath(directoryGradleGenerated)}\"") + + it.eachFile(FileType.FILES) { + if (it.name.endsWith(".kt")) { + def taskSourcePath = externalTestsProject.relativePath(it) + //remove filename extension. TODO make better replacement + def taskName = taskSourcePath.replace('/', '_').replace('-', '_').replaceFirst(~/\.[^\.]+$/, '') + directoryGradleGenerated.append("task $taskName (type: RunExternalTest) {\n source = \"$taskSourcePath\"\n}\n\n") + } } } - if (goldValue != null && goldValue != out.toString()) - throw new RuntimeException("test failed") - } - - private String llvmTool(String tool) { - return "${project.llvmDir}/bin/${tool}" - } - - protected List clangLinkArgs() { - return project.clangLinkArgs - } -} - -class RunKonanTest extends KonanTest { - void compileTest(String source, String exe) { - runCompiler(source, exe, []) - } -} - -class LinkKonanTest extends KonanTest { - protected String lib - - void compileTest(String source, String exe) { - def libDir = project.file(lib).absolutePath - def libBc = "${libDir}.bc" - - runCompiler(lib, libBc, ['-nolink', '-nostdlib']) - runCompiler(source, exe, ['-library', libBc]) } } @@ -109,6 +53,32 @@ task run() { dependsOn(tasks.withType(KonanTest).matching { it.enabled }) } +task run_external () { + doLast { + //TODO make better output / make as dependencies + def externalTestsProject = project.childProjects["external"] + def logFile = externalTestsProject.file("test-result.md") + logFile.write("|Test|Status|Comment|\n|----|------|-------|") + def current = 1 + def passed = 0 + def total = externalTestsProject.tasks.size() + externalTestsProject.tasks.each { + println("TEST: $current/$total (passed: $passed)") + try { + (it as RunExternalTest).executeTest() + logFile.append("\n|$it.name|PASSED||") + println("TEST PASSED") + passed++ + } catch (Exception ex) { + println("TEST FAILED") + logFile.append("\n|$it.name|FAILED|${ex.getMessage()}. Cause: ${ex.getCause()?.getMessage()}|") + } + current++ + } + logFile.append("\nPASSED: $passed/$total") + } +} + task sum (type:RunKonanTest) { source = "codegen/function/sum.kt" } diff --git a/backend.native/tests/external/build.gradle b/backend.native/tests/external/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy b/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy new file mode 100644 index 00000000000..9e0f8a2f703 --- /dev/null +++ b/buildSrc/src/main/groovy/org/jetbrains/kotlin/KonanTest.groovy @@ -0,0 +1,183 @@ +package org.jetbrains.kotlin + +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.TaskAction + +abstract class KonanTest extends DefaultTask { + protected String source + def backendNative = project.project(":backend.native") + def runtimeProject = project.project(":runtime") + def dist = project.rootProject.file("dist") + def llvmLlc = llvmTool("llc") + def runtimeBc = new File("${dist.canonicalPath}/lib/runtime.bc").absolutePath + def launcherBc = new File("${dist.canonicalPath}/lib/launcher.bc").absolutePath + def startKtBc = new File("${dist.canonicalPath}/lib/start.kt.bc").absolutePath + def stdlibKtBc = new File("${dist.canonicalPath}/lib/stdlib.kt.bc").absolutePath + def mainC = 'main.c' + String goldValue = null + String testData = null + List arguments = null + + boolean enabled = true + + public void setDisabled(boolean value) { + this.enabled = !value + } + + public KonanTest(){ + // TODO: that's a long reach up the project tree. + // May be we should reorganize a little. + //dependsOn(project.parent.parent.tasks['dist']) + dependsOn(project.rootProject.tasks['dist']) + } + + abstract void compileTest(List filesToCompile, String exe) + + protected void runCompiler(List filesToCompile, String output, List moreArgs) { + project.javaexec { + main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt' + classpath = project.configurations.cli_bc + jvmArgs "-ea", + "-Dkonan.home=${dist.canonicalPath}", + "-Djava.library.path=${dist.canonicalPath}/konan/nativelib" + args("-output", output, + *filesToCompile, + *moreArgs, + *project.globalArgs) + } + } + + protected void runCompiler(String source, String output, List moreArgs) { + runCompiler([source], output, moreArgs) + } + + String buildExePath() { + def exeName = project.file(source).name.replace(".kt", ".kt.exe") + def tempDir = temporaryDir.absolutePath + return "$tempDir/$exeName" + } + + List buildCompileList() { + return [project.file(source).absolutePath] + } + + @TaskAction + void executeTest() { + def exe = buildExePath() + + compileTest(buildCompileList(), exe) + println "execution :$exe" + + def out = null + project.exec { + commandLine exe + if (arguments != null) { + args arguments + } + if (testData != null) { + standardInput = new ByteArrayInputStream(testData.bytes) + } + if (goldValue != null) { + out = new ByteArrayOutputStream() + standardOutput = out + } + } + if (goldValue != null && goldValue != out.toString()) + throw new RuntimeException("test failed.") + } + + private String llvmTool(String tool) { + return "${project.llvmDir}/bin/${tool}" + } + + protected List clangLinkArgs() { + return project.clangLinkArgs + } +} + +class RunKonanTest extends KonanTest { + void compileTest(List filesToCompile, String exe) { + runCompiler(filesToCompile, exe, []) + } +} + +class LinkKonanTest extends KonanTest { + protected String lib + + void compileTest(List filesToCompile, String exe) { + def libDir = project.file(lib).absolutePath + def libBc = "${libDir}.bc" + + runCompiler(lib, libBc, ['-nolink', '-nostdlib']) + runCompiler(filesToCompile, exe, ['-library', libBc]) + } +} + +class RunExternalTest extends RunKonanTest { + + String goldValue = "OK" + + String buildExePath() { + def exeName = "${name}.kt.exe" + def tempDir = temporaryDir.absolutePath + return "$tempDir/$exeName" + } + + // TODO refactor + List buildCompileList() { + def result = [] + def filePattern = ~/(?m)\/\/\s*FILE:\s*(.*)$/ + def packagePattern = ~/(?m)package\s*([a-zA-z-][a-zA-Z0-9.-]*)/ //TODO check the regex + def boxPattern = ~/(?m)fun\s*box\s*\(\s*\)/ + def boxPackage = "" + def srcFile = project.file(source) + def srcText = srcFile.text + def matcher = filePattern.matcher(srcText) + def tmpDir = temporaryDir.absolutePath + + if (!matcher.find()) { + // There is only one file in the input + project.copy{ + from srcFile.absolutePath + into tmpDir + } + def newFile ="$tmpDir/${srcFile.name}" + if (srcText =~ boxPattern && srcText =~ packagePattern){ + boxPackage = (srcText =~ packagePattern)[0][1] + boxPackage += '.' + } + result.add(newFile) + } else { + // There are several files + def processedChars = 0 + while (true) { + def filePath = matcher.group(1) + filePath = "$tmpDir/$filePath" + def start = processedChars + def nextFileExists = matcher.find() + def end = nextFileExists ? matcher.start() : srcText.length() + def fileText = srcText.substring(start, end) + processedChars = end; + createFile(filePath, fileText) + if (fileText =~ boxPattern && fileText =~ packagePattern){ + boxPackage = (fileText =~ packagePattern)[0][1] + boxPackage += '.' + } + result.add(filePath) + if (!nextFileExists) break + } + } + createLauncherFile("$tmpDir/_launcher.kt", boxPackage) + result.add("$tmpDir/_launcher.kt") + return result + } + + void createLauncherFile(String file, String pkg) { + createFile(file, "fun main(args : Array) { print(${pkg}box()) }") + } + + void createFile(String file, String text) { + project.file(file).write(text) + } +} + diff --git a/settings.gradle b/settings.gradle index a4cb6759f92..de284fb5fbb 100644 --- a/settings.gradle +++ b/settings.gradle @@ -7,3 +7,4 @@ include ':backend.native' include ':runtime' include ':common' include ':backend.native:tests' +include ':backend.native:tests:external'