Move everything under kotlin-native folder

I was forced to manually do update the following files, because otherwise
they would be ignored according .gitignore settings. Probably they
should be deleted from repo.

Interop/.idea/compiler.xml
Interop/.idea/gradle.xml
Interop/.idea/libraries/Gradle__org_jetbrains_kotlin_kotlin_runtime_1_0_3.xml
Interop/.idea/libraries/Gradle__org_jetbrains_kotlin_kotlin_stdlib_1_0_3.xml
Interop/.idea/modules.xml
Interop/.idea/modules/Indexer/Indexer.iml
Interop/.idea/modules/Runtime/Runtime.iml
Interop/.idea/modules/StubGenerator/StubGenerator.iml
backend.native/backend.native.iml
backend.native/bc.frontend/bc.frontend.iml
backend.native/cli.bc/cli.bc.iml
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt
backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt
backend.native/tests/link/lib/foo.kt
backend.native/tests/link/lib/foo2.kt
backend.native/tests/teamcity-test.property
This commit is contained in:
Stanislav Erokhin
2020-10-27 21:00:28 +03:00
parent 91e4162dad
commit f624800b84
2830 changed files with 0 additions and 0 deletions
+114
View File
@@ -0,0 +1,114 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
import java.util.Properties
plugins {
// We explicitly configure versions of plugins in settings.gradle.kts.
// due to https://github.com/gradle/gradle/issues/1697.
id("kotlin")
groovy
`java-gradle-plugin`
}
buildscript {
dependencies {
classpath("com.google.code.gson:gson:2.8.6")
}
}
val rootProperties = Properties().apply {
rootDir.resolve("../gradle.properties").reader().use(::load)
}
val kotlinVersion: String by rootProperties
val kotlinCompilerRepo: String by rootProperties
val buildKotlinVersion: String by rootProperties
val buildKotlinCompilerRepo: String by rootProperties
val konanVersion: String by rootProperties
val slackApiVersion: String by rootProperties
val ktorVersion: String by rootProperties
val shadowVersion: String by rootProperties
val metadataVersion: String by rootProperties
group = "org.jetbrains.kotlin"
version = konanVersion
repositories {
jcenter()
maven(kotlinCompilerRepo)
maven(buildKotlinCompilerRepo)
maven("https://cache-redirector.jetbrains.com/maven-central")
mavenCentral()
maven("https://kotlin.bintray.com/kotlinx")
maven("https://dl.bintray.com/kotlin/kotlin-dev")
}
dependencies {
compileOnly(gradleApi())
implementation("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion")
implementation("org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion")
implementation("org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion")
implementation("com.ullink.slack:simpleslackapi:$slackApiVersion")
implementation("io.ktor:ktor-client-auth:$ktorVersion")
implementation("io.ktor:ktor-client-core:$ktorVersion")
implementation("io.ktor:ktor-client-cio:$ktorVersion")
api("org.jetbrains.kotlin:kotlin-native-utils:$kotlinVersion")
// Located in <repo root>/shared and always provided by the composite build.
api("org.jetbrains.kotlin:kotlin-native-shared:$konanVersion")
implementation("com.github.jengelman.gradle.plugins:shadow:$shadowVersion")
implementation("org.jetbrains.kotlinx:kotlinx-metadata-klib:$metadataVersion")
}
sourceSets["main"].withConvention(KotlinSourceSet::class) {
kotlin.srcDir("$projectDir/../tools/benchmarks/shared/src/main/kotlin/report")
}
gradlePlugin {
plugins {
create("benchmarkPlugin") {
id = "benchmarking"
implementationClass = "org.jetbrains.kotlin.benchmark.KotlinNativeBenchmarkingPlugin"
}
create("compileBenchmarking") {
id = "compile-benchmarking"
implementationClass = "org.jetbrains.kotlin.benchmark.CompileBenchmarkingPlugin"
}
create("swiftBenchmarking") {
id = "swift-benchmarking"
implementationClass = "org.jetbrains.kotlin.benchmark.SwiftBenchmarkingPlugin"
}
create("compileToBitcode") {
id = "compile-to-bitcode"
implementationClass = "org.jetbrains.kotlin.bitcode.CompileToBitcodePlugin"
}
create("runtimeTesting") {
id = "runtime-testing"
implementationClass = "org.jetbrains.kotlin.testing.native.RuntimeTestingPlugin"
}
}
}
val compileKotlin: KotlinCompile by tasks
val compileGroovy: GroovyCompile by tasks
// https://youtrack.jetbrains.com/issue/KT-37435
compileKotlin.apply {
kotlinOptions.jvmTarget = "1.8"
kotlinOptions.freeCompilerArgs += listOf("-Xno-optimized-callable-references", "-Xskip-prerelease-check")
}
// Add Kotlin classes to a classpath for the Groovy compiler
compileGroovy.apply {
classpath += project.files(compileKotlin.destinationDir)
dependsOn(compileKotlin)
}
@@ -0,0 +1,26 @@
pluginManagement {
val rootProperties = java.util.Properties().apply {
rootDir.resolve("../gradle.properties").reader().use(::load)
}
val kotlinCompilerRepo: String by rootProperties
val kotlinVersion by rootProperties
repositories {
maven(kotlinCompilerRepo)
maven("https://cache-redirector.jetbrains.com/maven-central")
mavenCentral()
}
resolutionStrategy {
eachPlugin {
if (requested.id.id == "kotlin") {
useModule("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion")
}
}
}
}
rootProject.name = "kotlin-native-build-tools"
includeBuild("../shared")
@@ -0,0 +1,579 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.JavaExec
import org.gradle.api.tasks.TaskAction
import org.gradle.process.ExecResult
import org.jetbrains.kotlin.utils.DFS
import java.nio.file.Paths
import java.util.function.Function
import java.util.function.UnaryOperator
import java.util.regex.Pattern
import java.util.stream.Collectors
class RunExternalTestGroup extends JavaExec {
def platformManager = project.rootProject.platformManager
def target = platformManager.targetManager(project.testTarget).target
def dist = UtilsKt.getKotlinNativeDist(project)
def enableKonanAssertions = true
String outputDirectory = null
String goldValue = null
// Checks test's output against gold value and returns true if the output matches the expectation
Function<String, Boolean> outputChecker = { str -> (goldValue == null || goldValue == str) }
boolean printOutput = true
String testData = null
int expectedExitStatus = 0
List<String> arguments = null
List<String> flags = null
boolean multiRuns = false
List<List<String>> multiArguments = null
boolean expectedFail = false
boolean compilerMessages = false
// Uses directory defined in $outputSourceSetName source set
void createOutputDirectory() {
if (outputDirectory != null) {
return
}
def outputSourceSet = UtilsKt.getTestOutputExternal(project)
outputDirectory = Paths.get(outputSourceSet, name).toString()
project.file(outputDirectory).mkdirs()
}
RunExternalTestGroup() {
// We don't build the compiler if a custom dist path is specified.
UtilsKt.dependsOnDist(this)
main = 'org.jetbrains.kotlin.cli.bc.K2NativeKt'
}
@Override
void exec() {
// Perhaps later we will return this exec() back but for now rest of infrastructure expects
// compilation begins on runCompiler call, to emulate this behaviour we call super.exec() after
// configuration part at runCompiler.
}
protected void runCompiler(List<String> filesToCompile, String output, List<String> moreArgs) {
def log = new ByteArrayOutputStream()
try {
classpath = project.fileTree("$dist.canonicalPath/konan/lib/") {
include '*.jar'
}
jvmArgs "-Xmx4G"
enableAssertions = true
def sources = File.createTempFile(name,".lst")
sources.deleteOnExit()
def sourcesWriter = sources.newWriter()
filesToCompile.each { f ->
sourcesWriter.write(f.chars().any { Character.isWhitespace(it) }
? "\"${f.replace("\\", "\\\\")}\"\n" // escape file name
: "$f\n")
}
sourcesWriter.close()
args = ["-output", output,
"@${sources.absolutePath}",
*moreArgs,
*project.globalTestArgs]
if (project.testTarget) {
args "-target", target.visibleName
}
if (enableKonanAssertions) {
args "-ea"
}
if (project.hasProperty("test_verbose")) {
println("Files to compile: $filesToCompile")
println(args)
}
standardOutput = log
errorOutput = log
super.exec()
} finally {
def logString = log.toString("UTF-8")
project.file("${output}.compilation.log").write(logString)
println(logString)
}
}
// FIXME: output directory here changes and hence this is not a property
String executablePath() { return "$outputDirectory/program.tr" }
OutputStream out
void runExecutable() {
if (!enabled) {
println "Test is disabled: $name"
return
}
def program = executablePath()
def suffix = target.family.exeSuffix
def exe = "$program.$suffix"
println "execution: $exe"
def compilerMessagesText = compilerMessages ? project.file("${program}.compilation.log").getText('UTF-8') : ""
out = new ByteArrayOutputStream()
//TODO Add test timeout
def times = multiRuns ? multiArguments.size() : 1
def exitCodeMismatch = false
for (int i = 0; i < times; i++) {
ExecResult execResult = project.execute {
commandLine exe
if (arguments != null) {
args arguments
}
if (multiRuns && multiArguments[i] != null) {
args multiArguments[i]
}
if (testData != null) {
standardInput = new ByteArrayInputStream(testData.bytes)
}
standardOutput = out
ignoreExitValue = true
}
exitCodeMismatch |= execResult.exitValue != expectedExitStatus
if (exitCodeMismatch) {
def message = "Expected exit status: $expectedExitStatus, actual: ${execResult.exitValue}"
if (this.expectedFail) {
println("Expected failure. $message")
} else {
throw new TestFailedException("Test failed on iteration $i. $message\n ${out.toString("UTF-8")}")
}
}
}
def result = compilerMessagesText + out.toString("UTF-8")
if (printOutput) {
println(result)
}
result = result.replace(System.lineSeparator(), "\n")
def goldValueMismatch = !outputChecker.apply(result)
if (goldValueMismatch) {
def message
if (goldValue != null) {
message = "Expected output: $goldValue, actual output: $result"
} else {
message = "Actual output doesn't match output checker: $result"
}
if (this.expectedFail) {
println("Expected failure. $message")
} else {
throw new TestFailedException("Test failed. $message")
}
}
if (!exitCodeMismatch && !goldValueMismatch && this.expectedFail) println("Unexpected pass")
}
/**
* If true, the test executable will be built in two stages:
* 1. Build a klibrary from sources.
* 2. Build a final executable from this klibrary.
*/
@Input
public def enableTwoStageCompilation = false
@Input
def groupDirectory = "."
String filter = project.findProperty("filter")
def testGroupReporter = new KonanTestGroupReportEnvironment(project)
void parseLanguageFlags(String src) {
def text = project.buildDir.toPath().resolve(src).text
def languageSettings = findLinesWithPrefixesRemoved(text, "// !LANGUAGE: ")
if (languageSettings.size() != 0) {
languageSettings.forEach { line ->
line.split(" ").toList().forEach { flags.add("-XXLanguage:$it") }
}
}
def experimentalSettings = findLinesWithPrefixesRemoved(text, "// !USE_EXPERIMENTAL: ")
if (experimentalSettings.size() != 0) {
experimentalSettings.forEach { line ->
line.split(" ").toList().forEach { flags.add("-Xopt-in=$it") }
}
}
def expectActualLinker = findLinesWithPrefixesRemoved(text, "// EXPECT_ACTUAL_LINKER")
if (expectActualLinker.size() != 0) {
flags.add("-Xexpect-actual-linker")
}
}
static String markMutableObjects(String text) {
def lines = text.readLines()
def result = new ArrayList<String>(lines.size())
lines.forEach { line ->
// FIXME: find only those who has vars inside
// Find object declarations and companion objects
if (line.matches("\\s*(private|public|internal)?\\s*object [a-zA-Z_][a-zA-Z0-9_]*\\s*.*")
|| line.matches("\\s*(private|public|internal)?\\s*companion object.*")) {
result += "@kotlin.native.ThreadLocal"
}
result += line
}
return result.join(System.lineSeparator())
}
static String insertInTextAfter(String text, String insert, String after) {
def begin = text.indexOf(after)
if (begin != -1) {
def end = text.indexOf("\n", begin)
text = text.substring(0, end) + insert + text.substring(end)
} else {
text = insert + text
}
return text
}
List<TestFile> createTestFiles(String src) {
def identifier = /[a-zA-Z_][a-zA-Z0-9_]/
def fullQualified = /[a-zA-Z_][a-zA-Z0-9_.]/
def importRegex = /(?m)^\s*import\s+/
def packagePattern = ~/(?m)^\s*package\s+(${fullQualified}*)/
def boxPattern = ~/(?m)fun\s+box\s*\(\s*\)/
def classPattern = ~/.*(class|object|enum|interface)\s+(${identifier}*).*/
def sourceName = "_" + normalize(project.buildDir.toPath().resolve(src).toFile().name)
def packages = new LinkedHashSet<String>()
def imports = []
def classes = []
def vars = new HashSet<String>() // variables that has the same name as a package
TestModule mainModule = null
def testFiles = TestDirectivesKt.buildCompileList(project.file("build/$src").toPath(), "$outputDirectory/$src")
for (TestFile testFile: testFiles) {
def text = testFile.text
def filePath = testFile.path
if (text.contains('COROUTINES_PACKAGE')) {
text = text.replace('COROUTINES_PACKAGE', 'kotlin.coroutines')
}
def pkg
if (text =~ packagePattern) {
pkg = (text =~ packagePattern)[0][1]
packages.add(pkg)
pkg = "$sourceName.$pkg"
text = text.replaceFirst(packagePattern, "package $pkg")
} else {
pkg = sourceName
text = insertInTextAfter(text, "\npackage $pkg\n", "@file:")
}
if (text =~ boxPattern) {
imports.add("${pkg}.*")
mainModule = testFile.module
}
// Find mutable objects that should be marked as ThreadLocal
if (filePath != "$outputDirectory/$src/helpers.kt") {
text = markMutableObjects(text)
}
testFile.text = text
}
for (TestFile testFile: testFiles) {
def text = testFile.text
// Find if there are any imports in the file
def matcher = (text =~ ~/${importRegex}(${fullQualified}*)/)
if (matcher) {
// Prepend package name to found imports
for (int i = 0; i < matcher.count; i++) {
String importStatement = matcher[i][1]
def subImport = importStatement.with {
int dotIdx = indexOf('.')
dotIdx > 0 ? substring(0, dotIdx) : it
}
if (packages.contains(subImport) || classes.contains(subImport)) {
// add only to those who import packages or import classes from the test files
text = text.replaceFirst(~/${importRegex}${Pattern.quote(importStatement)}/,
"import $sourceName.$importStatement")
} else if (text =~ classPattern) {
// special case for import from the local class
def clsMatcher = (text =~ classPattern)
for (int j = 0; j < clsMatcher.count; j++) {
def cl = (text =~ classPattern)[j][2]
classes.add(cl)
if (subImport == cl) {
text = text.replaceFirst(~/${importRegex}${Pattern.quote(importStatement)}/,
"import $sourceName.$importStatement")
}
}
}
}
} else if (packages.empty) {
// Add import statement after package
def pkg = null
if (text =~ packagePattern) {
pkg = 'package ' + (text =~ packagePattern)[0][1]
text = text.replaceFirst(packagePattern, '')
}
text = insertInTextAfter(text, (pkg ? "\n$pkg\n" : "") + "import $sourceName.*\n", "@file:")
}
// now replace all package usages in full qualified names
def res = "" // filesToCompile
text.eachLine { line ->
packages.each { pkg ->
// line contains val or var declaration or function parameter declaration
if ((line =~ ~/va(l|r) *$pkg*( *: *$fullQualified*)?( *get\(\))? *\=.*/) ||
(line =~ ~/fun .*\(\n?\s*$pkg:.*/)) {
vars.add(pkg)
}
if (line.contains("$pkg.") && !(line =~ packagePattern || line =~ importRegex)
&& !vars.contains(pkg)) {
def idx = 0
while ((idx = line.indexOf(pkg, idx)) >= 0) {
if (!Character.isJavaIdentifierPart(line.charAt(idx - 1))) {
line = line.substring(0, idx) + "$sourceName.$pkg" + line.substring(idx + pkg.length())
idx += sourceName.length() + pkg.length() + 1
} else {
idx += pkg.length()
}
}
}
}
res += "$line\n"
}
testFile.text = res
}
def launcherText = createLauncherFileText(src, imports)
testFiles.add(new TestFile("_launcher.kt", "$outputDirectory/$src/_launcher.kt".toString(),
launcherText, mainModule != null ? mainModule : TestModule.default))
return testFiles
}
String normalize(String name) {
name.replace('.kt', '')
.replace('-','_')
.replace('.', '_')
}
/**
* There are tests that require non-trivial 'package foo' in test launcher.
*/
String createLauncherFileText(String src, List<String> imports) {
StringBuilder text = new StringBuilder()
def pack = normalize(project.file(src).name)
text.append("package _$pack\n")
for (v in imports) {
text.append("import $v\n")
}
text.append(
"""
import kotlin.test.Test
@Test
fun runTest() {
@Suppress("UNUSED_VARIABLE")
val result = box()
if (result != "OK") throw AssertionError("Test failed with: " + result)
}
""" )
return text.toString()
}
List<String> findLinesWithPrefixesRemoved(String text, String prefix) {
def result = []
text.eachLine {
if (it.startsWith(prefix)) {
result.add(it - prefix)
}
}
return result
}
static def excludeList = [
"external/compiler/codegen/boxInline/multiplatform/defaultArguments/receiversAndParametersInLambda.kt", // KT-36880
"external/compiler/compileKotlinAgainstKotlin/specialBridgesInDependencies.kt", // KT-42723
"external/compiler/codegen/box/collections/kt41123.kt", // KT-42723
"external/compiler/codegen/box/multiplatform/multiModule/expectActualTypealiasLink.kt", // KT-40137
"external/compiler/codegen/box/multiplatform/multiModule/expectActualMemberLink.kt", // KT-33091
"external/compiler/codegen/box/multiplatform/multiModule/expectActualLink.kt", // KT-41901
"external/compiler/codegen/box/coroutines/multiModule/", // KT-40121
"external/compiler/codegen/box/callableReference/genericConstructorReference.kt", // KT-42631
"external/compiler/codegen/box/defaultArguments/recursiveDefaultArguments.kt" // KT-42684
]
boolean isEnabledForNativeBackend(String fileName) {
def text = project.buildDir.toPath().resolve(fileName).text
if (excludeList.any { fileName.replace(File.separator, "/").contains(it) }) return false
def languageSettings = findLinesWithPrefixesRemoved(text, '// !LANGUAGE: ')
if (!languageSettings.empty) {
def settings = languageSettings.first()
if (settings.contains('-ProperIeee754Comparisons') || // K/N supports only proper IEEE754 comparisons
settings.contains('-ReleaseCoroutines') || // only release coroutines
settings.contains('-DataClassInheritance') || // old behavior is not supported
settings.contains('-ProhibitAssigningSingleElementsToVarargsInNamedForm')) { // Prohibit these assignments
return false
}
}
def version = findLinesWithPrefixesRemoved(text, '// LANGUAGE_VERSION: ')
if (version.size() != 0 && (!version.contains("1.3") || !version.contains("1.4"))) {
// Support tests for 1.3 and exclude 1.2
return false
}
def apiVersion = findLinesWithPrefixesRemoved(text, '// !API_VERSION: ')
if (apiVersion.size() != 0 && !apiVersion.contains("1.4")) {
return false
}
def targetBackend = findLinesWithPrefixesRemoved(text, "// TARGET_BACKEND")
if (targetBackend.size() != 0) {
// There is some target backend. Check if it is NATIVE or not.
for (String s : targetBackend) {
if (s.contains("NATIVE")){ return true }
}
return false
} else {
// No target backend. Check if NATIVE backend is ignored.
def ignoredBackends = findLinesWithPrefixesRemoved(text, "// IGNORE_BACKEND: ")
for (String s : ignoredBackends) {
if (s.contains("NATIVE")) { return false }
}
// No ignored backends. Check if test is targeted to FULL_JDK or has JVM_TARGET set
if (!findLinesWithPrefixesRemoved(text, "// FULL_JDK").isEmpty()) { return false }
if (!findLinesWithPrefixesRemoved(text, "// JVM_TARGET:").isEmpty()) { return false }
return true
}
}
@TaskAction
void executeTest() {
createOutputDirectory()
// Form the test list.
List<File> ktFiles = project.buildDir.toPath().resolve(groupDirectory).toFile()
.listFiles({
it.isFile() && it.name.endsWith(".kt")
} as FileFilter)
if (filter != null) {
def pattern = ~filter
ktFiles = ktFiles.findAll {
it.name =~ pattern
}
}
testGroupReporter.suite(name) { suite ->
// Build tests in the group
flags = (flags ?: []) + "-tr"
List<TestFile> compileList = []
ktFiles.each {
def src = project.buildDir.relativePath(it)
if (isEnabledForNativeBackend(src)) {
// Create separate output directory for each test in the group.
project.file("$outputDirectory/${it.name}").mkdirs()
parseLanguageFlags(src)
compileList.addAll(createTestFiles(src))
}
}
compileList*.writeTextToFile()
try {
if (enableTwoStageCompilation) {
// Two-stage compilation.
def klibPath = "${executablePath()}.klib"
def files = compileList.stream()
.map { it.path }
.collect(Collectors.toList())
if (!files.empty) {
runCompiler(files, klibPath, flags + ["-p", "library"])
runCompiler([], executablePath(), flags + ["-Xinclude=$klibPath"])
}
} else {
// Regular compilation with modules.
Map<String, TestModule> modules = compileList.stream()
.map { it.module }
.distinct()
.collect(Collectors.toMap({ it.name }, UnaryOperator.identity() ))
List<TestModule> orderedModules = DFS.INSTANCE.topologicalOrder(modules.values()) { module ->
module.dependencies.collect { modules[it] }.findAll { it != null }
}
Set<String> libs = new HashSet<String>()
orderedModules.reverse().each { module ->
if (!module.isDefaultModule()) {
def klibModulePath = "${executablePath()}.${module.name}.klib"
libs.addAll(module.dependencies)
def klibs = libs.collectMany { ["-l", "${executablePath()}.${it}.klib"] }.toList()
def friends = module.friends ?
module.friends.collectMany {
["-friend-modules", "${executablePath()}.${it}.klib"]
}.toList() : []
runCompiler(compileList.findAll { it.module == module }.collect { it.path },
klibModulePath, flags + ["-p", "library"] + klibs + friends)
}
}
def compileMain = compileList.findAll {
it.module.isDefaultModule() || it.module == TestModule.support
}
compileMain.forEach { f ->
libs.addAll(f.module.dependencies)
}
def friends = compileMain.collectMany {it.module.friends }.toSet()
if (!compileMain.empty) {
runCompiler(compileMain.collect { it.path }, executablePath(), flags +
libs.collectMany { ["-l", "${executablePath()}.${it}.klib"] }.toList() +
friends.collectMany {["-friend-modules", "${executablePath()}.${it}.klib"]}.toList()
)
}
}
} catch (Exception ex) {
project.logger.quiet("ERROR: Compilation failed for test suite: $name with exception", ex)
project.logger.quiet("The following files were unable to compile:")
ktFiles.each { project.logger.quiet(it.name) }
suite.abort(ex, ktFiles.size())
throw new RuntimeException("Compilation failed", ex)
}
// Run the tests.
arguments = (arguments ?: []) + "--ktest_logger=SILENT"
ktFiles.each { file ->
def src = project.buildDir.relativePath(file)
def savedArgs = arguments
arguments += "--ktest_filter=_${normalize(file.name)}.*"
use(KonanTestSuiteReportKt) {
project.logger.quiet("TEST: $file.name " +
"(done: $testGroupReporter.statistics.total/${ktFiles.size()}, " +
"passed: $testGroupReporter.statistics.passed, " +
"skipped: $testGroupReporter.statistics.skipped)")
}
if (isEnabledForNativeBackend(src)) {
suite.executeTest(file.name) {
project.logger.quiet(src)
runExecutable()
}
} else {
suite.skipTest(file.name)
}
arguments = savedArgs
}
}
}
}
@@ -0,0 +1,313 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin
import org.gradle.api.Named
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.artifacts.Configuration
import org.gradle.api.file.FileCollection
import org.gradle.api.internal.AbstractNamedDomainObjectContainer
import org.gradle.api.internal.CollectionCallbackActionDecorator
import org.gradle.api.internal.file.AbstractFileCollection
import org.gradle.api.tasks.JavaExec
import org.gradle.api.tasks.SourceSet
import org.gradle.api.tasks.TaskDependency
import org.gradle.internal.reflect.Instantiator
import org.jetbrains.kotlin.konan.target.*
import org.jetbrains.kotlin.konan.util.*
class NamedNativeInteropConfig implements Named {
private final Project project
final String name
private String interopStubsName
private SourceSet interopStubs
final JavaExec genTask
private String flavor = "jvm"
private String defFile
private String pkg
private String target
private List<String> compilerOpts = []
private List<String> headers;
private String linker
List<String> linkerOpts = []
private FileCollection linkFiles;
private List<String> linkTasks = []
Configuration configuration
void flavor(String value) {
flavor = value
}
void defFile(String value) {
defFile = value
genTask.inputs.file(project.file(defFile))
}
void target(String value) {
target = value
}
void pkg(String value) {
pkg = value
}
void compilerOpts(List<String> values) {
compilerOpts.addAll(values)
}
void compilerOpts(String... values) {
compilerOpts.addAll(values)
}
void headers(FileCollection files) {
dependsOnFiles(files)
headers = headers + files.toSet().collect { it.absolutePath }
}
void headers(String... values) {
headers = headers + values.toList()
}
void linker(String value) {
linker = value
}
void linkerOpts(String... values) {
this.linkerOpts(values.toList())
}
void linkerOpts(List<String> values) {
linkerOpts.addAll(values)
}
void dependsOn(Object... deps) {
// TODO: add all files to inputs
genTask.dependsOn(deps)
}
private void dependsOnFiles(FileCollection files) {
dependsOn(files)
genTask.inputs.files(files)
}
void link(FileCollection files) {
linkFiles = linkFiles + files
dependsOnFiles(files)
}
void linkOutputs(Task task) {
linkOutputs(task.name)
}
void linkOutputs(String task) {
linkTasks += task
dependsOn(task)
final Project prj;
String taskName;
int index = task.lastIndexOf(':')
if (index != -1) {
prj = project.project(task.substring(0, index))
taskName = task.substring(index + 1)
} else {
prj = project
taskName = task
}
prj.tasks.matching { it.name == taskName }.all { // TODO: it is a hack
this.dependsOnFiles(it.outputs.files)
}
}
void includeDirs(String... values) {
compilerOpts.addAll(values.collect {"-I$it"})
}
File getNativeLibsDir() {
return new File(project.buildDir, "nativelibs/$target")
}
File getGeneratedSrcDir() {
return new File(project.buildDir, "nativeInteropStubs/$name/kotlin")
}
File getTemporaryFilesDir() {
return new File(project.buildDir, "interopTemp")
}
NamedNativeInteropConfig(Project project, String name, String target = null, String flavor = 'jvm') {
this.name = name
this.project = project
this.flavor = flavor
def platformManager = project.rootProject.ext.platformManager
def targetManager = platformManager.targetManager(target)
this.target = targetManager.targetName
this.headers = []
this.linkFiles = project.files()
interopStubsName = name + "InteropStubs"
genTask = project.task("gen" + interopStubsName.capitalize(), type: JavaExec)
this.configure()
}
private void configure() {
if (project.plugins.hasPlugin("kotlin")) {
interopStubs = project.sourceSets.create(interopStubsName)
configuration = project.configurations.create(interopStubs.name)
project.tasks.getByName(interopStubs.getTaskName("compile", "Kotlin")) {
dependsOn genTask
}
interopStubs.kotlin.srcDirs generatedSrcDir
project.dependencies {
add interopStubs.getCompileConfigurationName(), project(path: ':Interop:Runtime')
}
this.configuration.extendsFrom project.configurations[interopStubs.runtimeConfigurationName]
project.dependencies.add(this.configuration.name, interopStubs.output)
}
genTask.configure {
classpath = project.configurations.interopStubGenerator
main = "org.jetbrains.kotlin.native.interop.gen.jvm.MainKt"
jvmArgs '-ea'
systemProperties "java.library.path" : project.files(
new File(project.findProject(":Interop:Indexer").buildDir, "nativelibs"),
new File(project.findProject(":Interop:Runtime").buildDir, "nativelibs")
).asPath
// Set the konan.home property because we run the cinterop tool not from a distribution jar
// so it will not be able to determine this path by itself.
systemProperties "konan.home": project.rootProject.projectDir
environment "LIBCLANG_DISABLE_CRASH_RECOVERY": "1"
outputs.dir generatedSrcDir
outputs.dir nativeLibsDir
outputs.dir temporaryFilesDir
// defer as much as possible
doFirst {
List<String> linkerOpts = this.linkerOpts
linkTasks.each {
linkerOpts += project.tasks.getByPath(it).outputs.files.files
}
linkerOpts += linkFiles.files
args '-generated', generatedSrcDir
args '-natives', nativeLibsDir
args '-Xtemporary-files-dir', temporaryFilesDir
args '-flavor', this.flavor
if (flavor == "jvm") {
args '-mode', 'sourcecode'
}
// Uncomment to debug.
// args '-verbose', 'true'
if (defFile != null) {
args '-def', project.file(defFile)
}
if (pkg != null) {
args '-pkg', pkg
}
if (linker != null) {
args '-linker', linker
}
if (target != null) {
args '-target', target
}
// TODO: the interop plugin should probably be reworked to execute clang from build scripts directly
environment['PATH'] = project.files(project.hostPlatform.clang.clangPaths).asPath +
File.pathSeparator + environment['PATH']
args compilerOpts.collectMany { ['-compiler-option', it] }
args linkerOpts.collectMany { ['-linker-option', it] }
headers.each {
args '-header', it
}
}
}
}
}
class NativeInteropExtension extends AbstractNamedDomainObjectContainer<NamedNativeInteropConfig> {
private final Project project
private String target = null
private String flavor = 'jvm'
protected NativeInteropExtension(Project project) {
super(NamedNativeInteropConfig, project.gradle.services.get(Instantiator), project.gradle.services.get(CollectionCallbackActionDecorator))
this.project = project
}
@Override
protected NamedNativeInteropConfig doCreate(String name) {
def config = new NamedNativeInteropConfig(project, name, target, flavor)
return config
}
public void target(String value) {
this.target = value
}
public void flavor(String value) {
this.flavor = value
}
}
class NativeInteropPlugin implements Plugin<Project> {
@Override
void apply(Project prj) {
prj.extensions.add("kotlinNativeInterop", new NativeInteropExtension(prj))
def runtimeNativeLibsDir = new File(prj.findProject(':Interop:Runtime').buildDir, 'nativelibs')
def nativeLibsDir = new File(prj.buildDir, "nativelibs")
prj.configurations {
interopStubGenerator
}
prj.dependencies {
interopStubGenerator project(path: ":Interop:StubGenerator")
interopStubGenerator project(path: ":endorsedLibraries:kotlinx.cli", configuration: "jvmRuntimeElements")
}
}
}
@@ -0,0 +1,6 @@
package org.jetbrains.kotlin
enum class BenchmarkRepeatingType {
INTERNAL, // Let the benchmark perform warmups and repeats.
EXTERNAL, // Repeat by relaunching benchmark
}
@@ -0,0 +1,88 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.TaskAction
import java.io.BufferedReader
import java.io.FileInputStream
import java.io.InputStreamReader
import java.io.OutputStreamWriter
import java.net.HttpURLConnection
import java.net.URL
import java.util.*
/**
* Task to save benchmarks results on server.
*
* @property bundleSize size of build
* @property onlyBranch register only builds for branch
* @property fileWithResult json file with benchmarks run results
*/
open class BuildRegister : DefaultTask() {
var onlyBranch: String? = null
var bundleSize: Int? = null
var fileWithResult: String = "nativeReport.json"
val performanceServer = "https://kotlin-native-perf-summary.labs.jb.gg"
private fun sendPostRequest(url: String, body: String): String {
val connection = URL(url).openConnection() as HttpURLConnection
return connection.apply {
setRequestProperty("Content-Type", "application/json; charset=utf-8")
requestMethod = "POST"
doOutput = true
val outputWriter = OutputStreamWriter(outputStream)
outputWriter.write(body)
outputWriter.flush()
}.let {
if (it.responseCode == 200) it.inputStream else it.errorStream
}.let { streamToRead ->
BufferedReader(InputStreamReader(streamToRead)).use {
val response = StringBuffer()
var inputLine = it.readLine()
while (inputLine != null) {
response.append(inputLine)
inputLine = it.readLine()
}
it.close()
response.toString()
}
}
}
@TaskAction
fun run() {
// Get TeamCity properties.
val teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") ?: error("Can't load teamcity config!")
val buildProperties = Properties()
buildProperties.load(FileInputStream(teamcityConfig))
val buildId = buildProperties.getProperty("teamcity.build.id")
val teamCityUser = buildProperties.getProperty("teamcity.auth.userId")
val teamCityPassword = buildProperties.getProperty("teamcity.auth.password")
// Get branch.
val currentBuild = getBuild("id:$buildId", teamCityUser, teamCityPassword)
val branch = getBuildProperty(currentBuild, "branchName")
// Send post request to register build.
val requestBody = buildString {
append("{\"buildId\":\"$buildId\",")
append("\"teamCityUser\":\"$teamCityUser\",")
append("\"teamCityPassword\":\"$teamCityPassword\",")
append("\"fileWithResult\":\"$fileWithResult\",")
append("\"bundleSize\": ${bundleSize?.let { "\"$bundleSize\"" } ?: bundleSize}}")
}
if (onlyBranch == null || onlyBranch == branch) {
println(sendPostRequest("$performanceServer/register", requestBody))
} else {
println("Skipping registration. Current branch $branch, need registration for $onlyBranch!")
}
}
}
@@ -0,0 +1,60 @@
package org.jetbrains.kotlin
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.tasks.Exec
import org.jetbrains.kotlin.konan.target.CompilerOutputKind
import org.jetbrains.kotlin.konan.util.visibleName
class CacheTesting(val buildCacheTask: Task, val compilerArgs: List<String>, val isDynamic: Boolean)
fun configureCacheTesting(project: Project): CacheTesting? {
val cacheKindString = project.findProperty("test_with_cache_kind") as String? ?: return null
val isDynamic = when (cacheKindString) {
"dynamic" -> true
"static" -> false
else -> error(cacheKindString)
}
val cacheKind = if (isDynamic) {
CompilerOutputKind.DYNAMIC_CACHE
} else {
CompilerOutputKind.STATIC_CACHE
}
val target = project.testTarget
val cacheDir = project.file("${project.buildDir}/cache")
val cacheFile = "$cacheDir/stdlib-cache"
val dist = project.kotlinNativeDist
val stdlib = "$dist/klib/common/stdlib"
val compilerArgs = listOf("-Xcached-library=$stdlib,$cacheFile")
val buildCacheTask = project.tasks.create("buildStdlibCache", Exec::class.java) {
it.doFirst {
cacheDir.mkdirs()
}
if (!(project.property("useCustomDist") as Boolean)) {
val tasks = listOf(
"${target}CrossDist",
"${target}CrossDistRuntime",
"distCompiler"
).map { task -> project.rootProject.tasks.getByName(task) }
it.dependsOn(tasks)
}
it.commandLine(
"$dist/bin/konanc",
"-p", cacheKind.visibleName,
"-o", "$cacheDir/stdlib-cache",
"-Xmake-cache=$stdlib",
"-no-default-libs", "-nostdlib",
"-target", target,
"-g"
)
}
return CacheTesting(buildCacheTask, compilerArgs, isDynamic)
}
@@ -0,0 +1,85 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.TaskAction
import org.gradle.api.artifacts.Configuration
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFiles
import java.io.File
import java.util.zip.ZipFile
/**
* Task to find out collisions before producing fat jar.
*
* @property configurations added to fat jar configurations
* @property resolvingRules a map containing rules to resolve conflicts. Key - conflicting file, value - a jar to copy the file from
* @property resolvingRulesWithRegexes a map containing rules to resolve conflicts. Key - regular expression describing conflicting file, value - a jar to copy the file from
* @property librariesWithIgnoredClassCollisions libraries which collision in class files are ignored
*/
open class CollisionDetector : DefaultTask() {
@InputFiles
var configurations = listOf<Configuration>()
@Input
val resolvingRules = mutableMapOf<String, String>()
@Input
val resolvingRulesWithRegexes = mutableMapOf<Regex, String>()
@Input
val librariesWithIgnoredClassCollisions = mutableListOf<String>()
val resolvedConflicts = mutableMapOf<String, File>()
// Key - filename, value - jar file containing it.
private val filesInfo = mutableMapOf<String, String>()
@TaskAction
fun run() {
configurations.forEach { configuration ->
configuration.files.filter { it.name.endsWith(".jar") }.forEach { processedFile ->
ZipFile(processedFile).use { zip ->
zip.entries().asSequence().filterNot {
it.isDirectory ||
it.name.equals("META-INF/MANIFEST.MF", ignoreCase = true) ||
it.name.equals("META-INF/versions/9/module-info.class", ignoreCase = true) ||
it.name.startsWith("META-INF/services/", ignoreCase = true)
}.forEach {
val outputPath = it.name
if (outputPath in filesInfo.keys) {
val rule = resolvingRules.getOrElse(outputPath) {
resolvingRulesWithRegexes.entries.firstOrNull { (key, _) -> key.matches(outputPath) }?.value
}
var ignoreJar = false
if (rule != null && processedFile.name.startsWith(rule)) {
resolvedConflicts[outputPath] = processedFile
} else {
// Skip class files from ignored libraries if version of libraries had collision are the same.
val versionRegex = "\\d+\\.\\d+(\\.\\d+)?(-M\\d)?(-\\w+(-\\d+)?)?".toRegex()
val currentVersion = versionRegex.find(processedFile.name)?.groupValues?.get(0)
val collisionLibVersion = versionRegex.find(filesInfo.getValue(outputPath))?.groupValues?.get(0)
if (outputPath.endsWith(".class") && currentVersion == collisionLibVersion) {
if (processedFile.name == filesInfo[outputPath]) {
ignoreJar = true
} else {
librariesWithIgnoredClassCollisions.forEach {
if (processedFile.name.startsWith(it)) {
ignoreJar = true
}
}
}
}
}
if (rule == null && !ignoreJar) {
error("Collision is detected. File $outputPath is found in ${filesInfo[outputPath]} and ${processedFile.name}")
}
} else {
filesInfo[outputPath] = processedFile.name
}
}
}
}
}
}
}
@@ -0,0 +1,48 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer
import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext
import org.gradle.api.file.FileTreeElement
import shadow.org.apache.tools.zip.ZipOutputStream
import java.io.File
import shadow.org.apache.commons.io.IOUtils
import shadow.org.apache.tools.zip.ZipEntry
import shadow.org.apache.tools.zip.ZipFile
class CollisionTransformer: Transformer {
var resolvedConflicts = mutableMapOf<String, File>()
private val foundConflictsFiles = mutableSetOf<String>()
override fun canTransformResource(element: FileTreeElement): Boolean {
val result = element.name in resolvedConflicts.keys
if (result) {
foundConflictsFiles.add(element.name)
}
return result
}
override fun transform(context: TransformerContext) {}
override fun hasTransformedResource(): Boolean {
return foundConflictsFiles.isNotEmpty()
}
override fun modifyOutputStream(jos: ZipOutputStream, preserveFileTimestamps: Boolean) {
foundConflictsFiles.forEach {
val entry = ZipEntry(it)
entry.time = TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.time)
jos.putNextEntry(entry)
val archive = ZipFile(resolvedConflicts[it])
archive.getInputStream(archive.getEntry(it)).use {
IOUtils.copyLarge(it, jos)
}
jos.closeEntry()
}
foundConflictsFiles.clear()
}
}
@@ -0,0 +1,129 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin
import java.io.File
import javax.inject.Inject
import org.gradle.api.GradleException
import com.google.gson.annotations.Expose
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.OutputFile
import org.gradle.api.tasks.TaskAction
import org.jetbrains.kotlin.bitcode.CompileToBitcode
import java.io.FileReader
import java.io.FileWriter
internal data class Entry(
@Expose val directory: String,
@Expose val file: String,
@Expose val arguments: List<String>,
@Expose val output: String
) {
companion object {
fun create(
directory: File,
file: File,
args: List<String>,
outputDir: File
): Entry {
return Entry(
directory.absolutePath,
file.absolutePath,
args + listOf(file.absolutePath),
File(outputDir, file.name + ".o").absolutePath
)
}
fun writeListTo(file: File, entries: List<Entry>) = FileWriter(file).use {
gson.toJson(entries, it)
}
fun readListFrom(file: File): Array<Entry> = FileReader(file).use {
gson.fromJson(it, Array<Entry>::class.java)
}
}
}
open class GenerateCompilationDatabase @Inject constructor(@Input val target: String,
@Input val srcRoot: File,
@Input val files: Iterable<File>,
@Input val executable: String,
@Input val compilerFlags: List<String>,
@Input val outputDir: File
) : DefaultTask() {
@OutputFile
var outputFile = File(outputDir, "compile_commands.json")
@TaskAction
fun run() {
val plugin = project.convention.getPlugin(ExecClang::class.java)
val executable = plugin.resolveExecutable(executable)
val args = listOf(executable) + compilerFlags + plugin.konanArgs(target)
val entries: List<Entry> = files.map { Entry.create(srcRoot, it, args, outputDir) }
Entry.writeListTo(outputFile, entries)
}
}
open class MergeCompilationDatabases @Inject constructor() : DefaultTask() {
@InputFiles
val inputFiles = mutableListOf<File>()
@OutputFile
var outputFile = File(project.buildDir, "compile_commands.json")
@TaskAction
fun run() {
val entries = mutableListOf<Entry>()
for (file in inputFiles) {
entries.addAll(Entry.readListFrom(file))
}
Entry.writeListTo(outputFile, entries)
}
}
fun mergeCompilationDatabases(project: Project, name: String, paths: List<String>): Task {
val subtasks: List<MergeCompilationDatabases> = paths.map {
val task = project.tasks.getByPath(it)
if (task !is MergeCompilationDatabases) {
throw GradleException("Unknown task type for compdb merging: $task")
}
task
}
return project.tasks.create(name, MergeCompilationDatabases::class.java) { task ->
task.dependsOn(subtasks)
task.inputFiles.addAll(subtasks.map { it.outputFile })
}
}
/**
* Get all [CompileToBitcode] tasks in the project, group them by target, and generate
* compilation database for each target. Tasks will be named <target>[name]. Databases
* will be placed in a build dir in <target>/compile_commands.json
*/
fun createCompilationDatabasesFromCompileToBitcodeTasks(project: Project, name: String) {
val compileTasks = project.tasks.withType(CompileToBitcode::class.java).toList()
val compdbTasks = compileTasks.groupBy({ task -> task.target }) { task ->
project.tasks.create("${task.name}_CompilationDatabase",
GenerateCompilationDatabase::class.java,
task.target,
task.srcRoot,
task.inputFiles,
task.executable,
task.compilerFlags,
task.objDir)
}
for ((target, tasks) in compdbTasks) {
project.tasks.create("${target}${name}", MergeCompilationDatabases::class.java) { task ->
task.dependsOn(tasks)
task.inputFiles.addAll(tasks.map { it.outputFile })
task.outputFile = File(File(project.buildDir, target), "compile_commands.json")
}
}
}
@@ -0,0 +1,79 @@
package org.jetbrains.kotlin
import org.gradle.api.DefaultTask
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFiles
import org.gradle.api.tasks.OutputDirectory
import org.gradle.api.tasks.TaskAction
import java.io.File
open class CopyCommonSources : DefaultTask() {
@Input
var zipSources: Boolean = false
@InputFiles
var sourcePaths: ConfigurableFileCollection = project.files()
@OutputDirectory
var outputDir: File = project.buildDir.resolve("sources")
fun zipSources(needToZip: Boolean) {
zipSources = needToZip
}
fun outputDir(path: Any) {
outputDir = project.file(path)
}
fun sourcePaths(paths: Any) {
sourcePaths = project.files(paths)
}
@TaskAction
fun copySources() {
if (zipSources) copyAndZip() else copyPlain()
}
private fun copyPlain() {
for (sourcePath in sourcePaths) {
sourcePath.copyFilteredTo(outputDir)
}
}
private fun copyAndZip() {
for (sourcePath in sourcePaths) {
val filePrefix = sourcePath.name.replace(Regex("-\\d+.*"), "")
val targetFileName = "$filePrefix-sources.zip"
val tempDir = project.buildDir.resolve(name).resolve(filePrefix).also {
it.deleteRecursively()
it.mkdirs()
}
sourcePath.copyFilteredTo(tempDir)
project.ant.invokeMethod(
"zip",
mapOf(
"destfile" to outputDir.resolve(targetFileName).absolutePath,
"basedir" to tempDir.absolutePath
)
)
}
}
private fun File.copyFilteredTo(destinationDir: File) {
val fileTree = if (isFile) project.zipTree(this) else project.fileTree(this)
project.copy {
it.from(fileTree)
it.includeEmptyDirs = false
it.include("generated/**/*.kt")
it.include("kotlin/**/*.kt")
it.include("kotlin.test/*.kt")
it.into(destinationDir)
}
}
}
@@ -0,0 +1,55 @@
package org.jetbrains.kotlin
import groovy.lang.Closure
import org.gradle.api.Task
import org.gradle.api.tasks.Copy
/**
* A task that copies samples and replaces direct repository URLs with ones provided by the cache-redirector service.
*/
open class CopySamples: Copy() {
var samplesDir = project.file("samples")
init {
configureReplacements()
}
fun samplesDir(path: Any) {
samplesDir = project.file(path)
}
private fun configureReplacements() {
from(samplesDir) {
it.exclude("**/*.gradle")
}
from(samplesDir) {
it.include("**/*.gradle")
it.filter { line ->
replacements.forEach { (repo, replacement) ->
if (line.contains(repo)) {
return@filter line.replace(repo, replacement)
}
}
return@filter line
}
}
}
override fun configure(closure: Closure<Any>): Task {
super.configure(closure)
configureReplacements()
return this
}
companion object {
val replacements = listOf(
"mavenCentral()" to "maven { url 'https://cache-redirector.jetbrains.com/maven-central' }",
"jcenter()" to "maven { url 'https://cache-redirector.jetbrains.com/jcenter' }",
"https://dl.bintray.com/kotlin/kotlin-dev" to "https://cache-redirector.jetbrains.com/dl.bintray.com/kotlin/kotlin-dev",
"https://dl.bintray.com/kotlin/kotlin-eap" to "https://cache-redirector.jetbrains.com/dl.bintray.com/kotlin/kotlin-eap",
"https://dl.bintray.com/kotlin/ktor" to "https://cache-redirector.jetbrains.com/dl.bintray.com/kotlin/ktor",
"https://plugins.gradle.org/m2" to "https://cache-redirector.jetbrains.com/plugins.gradle.org/m2"
)
}
}
@@ -0,0 +1,138 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin
import groovy.lang.Closure
import org.gradle.api.DefaultTask
import org.gradle.api.Task
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.TaskAction
import org.jetbrains.kotlin.konan.target.AppleConfigurables
/**
* Test task for -Xcoverage and -Xlibraries-to-cover flags. Requires a binary to be built by the Konan plugin
* with
* konanArtifacts {
* program([binaryName], targets: [testTarget]) {
* ...
* extraOpts "-Xcoverage"/"-Xlibrary-to-cover=...", "-Xcoverage-file=$[profrawFile]"
* }
* }
* and a dependency set according to a pattern "run${binaryName}".
*
* @property numberOfCoveredFunctions Expected number of covered functions
* @property numberOfCoveredLines Expected number of covered lines in all functions
* @property binaryName Name of the produced binary
*/
open class CoverageTest : DefaultTask() {
private val target = project.testTarget
private val platform = project.platformManager.platform(target)
private val configurables = platform.configurables
// Use the same LLVM version as compiler when producing machine code:
private val llvmToolsDir = if (configurables is AppleConfigurables) {
"${configurables.absoluteTargetToolchain}/usr/bin"
} else {
"${configurables.absoluteLlvmHome}/bin"
}
@Input
lateinit var binaryName: String
// TODO: Consider better metric.
@Input
var numberOfCoveredFunctions: Int? = null
@Input
var numberOfCoveredLines: Int? = null
val profrawFile: String by lazy {
"${project.buildDir.absolutePath}/$binaryName.profraw"
}
private val profdataFile: String by lazy {
"${project.buildDir.absolutePath}/$binaryName.profdata"
}
private val outputDir: String by lazy {
project.file(project.property("testOutputCoverage")!!).absolutePath
}
override fun configure(closure: Closure<Any>): Task {
super.configure(closure)
dependsOnDist()
dependsOn(project.tasks.getByName("compileKonan$binaryName"))
return this
}
@TaskAction
fun run() {
val suffix = target.family.exeSuffix
val pathToBinary = "$outputDir/$binaryName/$target/$binaryName.$suffix"
runProcess({ project.executor.execute(it) }, pathToBinary)
.ensureSuccessful(pathToBinary)
exec("llvm-profdata", "merge", profrawFile, "-o", profdataFile)
val llvmCovResult = exec("llvm-cov", "export", pathToBinary, "-instr-profile", profdataFile)
val jsonReport = llvmCovResult.stdOut
val llvmCovReport = parseLlvmCovReport(jsonReport)
try {
CoverageValidator(numberOfCoveredFunctions, numberOfCoveredLines).validateReport(llvmCovReport)
} catch (e: TestFailedException) {
// Show report in message to make debug easier.
val show = exec("llvm-cov", "show", pathToBinary, "-instr-profile", profdataFile).stdOut
// llvm-cov output contains '|' so another symbol is used as margin prefix.
throw TestFailedException("""
>${e.message}
>$show
""".trimMargin(">"))
}
}
private fun exec(llvmTool: String, vararg args: String): ProcessOutput {
val executable = "$llvmToolsDir/$llvmTool"
val result = runProcess(localExecutor(project), executable, args.toList())
result.ensureSuccessful(llvmTool)
return result
}
private fun ProcessOutput.ensureSuccessful(executable: String) {
if (exitCode != 0) {
println("""
$executable failed.
exitCode: $exitCode
stdout:
$stdOut
stderr:
$stdErr
""".trimIndent())
error("$executable failed")
}
}
}
private class CoverageValidator(
val numberOfCoveredFunctions: Int?,
val numberOfCoveredLines: Int?
) {
fun validateReport(report: LlvmCovReport) {
val data = report.data
if (data.isEmpty()) {
failTest("Report data should not be empty!")
}
compareNumbers(numberOfCoveredFunctions, data[0].totals.functions.covered, "Number of covered functions")
compareNumbers(numberOfCoveredLines, data[0].totals.lines.covered, "Number of covered lines")
}
private fun compareNumbers(expected: Int?, actual: Int, description: String) {
if (expected != null && actual != expected) {
failTest("$description differs from expected! Expected: $expected. Got: $actual")
}
}
private fun failTest(message: String) {
throw TestFailedException(message)
}
}
@@ -0,0 +1,13 @@
package org.jetbrains.kotlin
import org.gradle.api.Project
data class EndorsedLibraryInfo(val project: Project, val name: String) {
val projectName: String
get() = project.name
val taskName: String by lazy {
projectName.split('.').joinToString(separator = "") { it.capitalize() }
}
}
@@ -0,0 +1,107 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin
import org.gradle.api.Action
import groovy.lang.Closure
import org.gradle.api.GradleException
import org.gradle.api.Project
import org.gradle.process.ExecResult
import org.gradle.process.ExecSpec
import org.gradle.util.ConfigureUtil
import org.jetbrains.kotlin.konan.target.*
import org.jetbrains.kotlin.konan.file.*
class ExecClang(private val project: Project) {
private val platformManager = project.rootProject.findProperty("platformManager") as PlatformManager
private fun konanArgs(target: KonanTarget): List<String> {
return platformManager.platform(target).clang.clangArgsForKonanSources.asList()
}
fun konanArgs(targetName: String?): List<String> {
val target = platformManager.targetManager(targetName).target
return konanArgs(target)
}
fun resolveExecutable(executable: String?): String {
val executable = executable ?: "clang"
if (listOf("clang", "clang++").contains(executable)) {
val llvmDir = project.findProperty("llvmDir")
return "${llvmDir}/bin/$executable"
} else {
throw GradleException("unsupported clang executable: $executable")
}
}
// The bare ones invoke clang with system default sysroot.
fun execBareClang(action: Action<in ExecSpec>): ExecResult {
return this.execClang(emptyList<String>(), action)
}
fun execBareClang(closure: Closure<in ExecSpec>): ExecResult {
return this.execClang(emptyList<String>(), closure)
}
// The konan ones invoke clang with konan provided sysroots.
// So they require a target or assume it to be the host.
// The target can be specified as KonanTarget or as a
// (nullable, which means host) target name.
fun execKonanClang(target: String?, action: Action<in ExecSpec>): ExecResult {
return this.execClang(konanArgs(target), action)
}
fun execKonanClang(target: KonanTarget, action: Action<in ExecSpec>): ExecResult {
return this.execClang(konanArgs(target), action)
}
fun execKonanClang(target: String?, closure: Closure<in ExecSpec>): ExecResult {
return this.execClang(konanArgs(target), closure)
}
fun execKonanClang(target: KonanTarget, closure: Closure<in ExecSpec>): ExecResult {
return this.execClang(konanArgs(target), closure)
}
// These ones are private, so one has to choose either Bare or Konan.
private fun execClang(defaultArgs: List<String>, closure: Closure<in ExecSpec>): ExecResult {
return this.execClang(defaultArgs, ConfigureUtil.configureUsing(closure))
}
private fun execClang(defaultArgs: List<String>, action: Action<in ExecSpec>): ExecResult {
val extendedAction = object : Action<ExecSpec> {
override fun execute(execSpec: ExecSpec) {
action.execute(execSpec)
execSpec.apply {
executable = resolveExecutable(executable)
val hostPlatform = project.findProperty("hostPlatform") as Platform
environment["PATH"] = project.files(hostPlatform.clang.clangPaths).asPath +
java.io.File.pathSeparator + environment["PATH"]
args(defaultArgs)
}
}
}
return project.exec(extendedAction)
}
}
@@ -0,0 +1,571 @@
/*
* Copyright 2010-2018 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin
import com.google.gson.annotations.Expose
import groovy.lang.Closure
import org.gradle.api.Action
import org.gradle.api.Project
import org.gradle.process.ExecResult
import org.gradle.process.ExecSpec
import org.gradle.util.ConfigureUtil
import org.jetbrains.kotlin.konan.target.Architecture
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.target.Xcode
import java.io.*
import java.nio.file.Path
import java.nio.file.Paths
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
import java.util.concurrent.TimeUnit
/**
* A replacement of the standard `exec {}`
* @see org.gradle.api.Project.exec
*/
interface ExecutorService {
fun execute(closure: Closure<in ExecSpec>): ExecResult? = execute(ConfigureUtil.configureUsing(closure))
fun execute(action: Action<in ExecSpec>): ExecResult?
}
/**
* Creates an ExecutorService depending on a test target -Ptest_target
*/
fun create(project: Project): ExecutorService {
val platformManager = project.platformManager
val testTarget = project.testTarget
val platform = platformManager.platform(testTarget)
val absoluteTargetToolchain = platform.absoluteTargetToolchain
val absoluteTargetSysRoot = platform.absoluteTargetSysRoot
return when (testTarget) {
KonanTarget.WASM32 -> object : ExecutorService {
override fun execute(action: Action<in ExecSpec>): ExecResult? = project.exec { execSpec ->
action.execute(execSpec)
with(execSpec) {
val exe = executable
val d8 = "$absoluteTargetToolchain/bin/d8"
val launcherJs = "$executable.js"
executable = d8
args = listOf("--expose-wasm", launcherJs, "--", exe) + args
}
}
}
KonanTarget.LINUX_MIPS32, KonanTarget.LINUX_MIPSEL32 -> object : ExecutorService {
override fun execute(action: Action<in ExecSpec>): ExecResult? = project.exec { execSpec ->
action.execute(execSpec)
with(execSpec) {
val qemu = if (platform.target === KonanTarget.LINUX_MIPS32) "qemu-mips" else "qemu-mipsel"
val absoluteQemu = "$absoluteTargetToolchain/bin/$qemu"
val exe = executable
executable = absoluteQemu
args = listOf("-L", absoluteTargetSysRoot,
// This is to workaround an endianess issue.
// See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=731082 for details.
"$absoluteTargetSysRoot/lib/ld.so.1", "--inhibit-cache",
exe) + args
}
}
}
KonanTarget.IOS_X64,
KonanTarget.TVOS_X64,
KonanTarget.WATCHOS_X86,
KonanTarget.WATCHOS_X64 -> simulator(project)
KonanTarget.IOS_ARM32,
KonanTarget.IOS_ARM64 -> deviceLauncher(project)
else -> {
if (project.hasProperty("remote")) sshExecutor(project)
else localExecutorService(project)
}
}
}
data class ProcessOutput(var stdOut: String, var stdErr: String, var exitCode: Int)
/**
* Runs process using a given executor.
*
* @param executor a method that is able to run a given executable, e.g. ExecutorService::execute
* @param executable a process executable to be run
* @param args arguments for a process
*/
fun runProcess(executor: (Action<in ExecSpec>) -> ExecResult?,
executable: String, args: List<String>): ProcessOutput {
val outStream = ByteArrayOutputStream()
val errStream = ByteArrayOutputStream()
val execResult = executor(Action {
it.executable = executable
it.args = args.toList()
it.standardOutput = outStream
it.errorOutput = errStream
it.isIgnoreExitValue = true
})
checkNotNull(execResult)
val stdOut = outStream.toString("UTF-8")
val stdErr = errStream.toString("UTF-8")
return ProcessOutput(stdOut, stdErr, execResult.exitValue)
}
fun runProcess(executor: (Action<in ExecSpec>) -> ExecResult?,
executable: String, vararg args: String) = runProcess(executor, executable, args.toList())
/**
* Runs process using a given executor.
*
* @param executor a method that is able to run a given executable, e.g. ExecutorService::execute
* @param executable a process executable to be run
* @param args arguments for a process
* @param input an input string to be passed through the standard input stream
*/
fun runProcessWithInput(executor: (Action<in ExecSpec>) -> ExecResult?,
executable: String, args: List<String>, input: String): ProcessOutput {
val outStream = ByteArrayOutputStream()
val errStream = ByteArrayOutputStream()
val inStream = ByteArrayInputStream(input.toByteArray())
val execResult = executor(Action {
it.executable = executable
it.args = args.toList()
it.standardOutput = outStream
it.errorOutput = errStream
it.isIgnoreExitValue = true
it.standardInput = inStream
})
checkNotNull(execResult)
val stdOut = outStream.toString("UTF-8")
val stdErr = errStream.toString("UTF-8")
return ProcessOutput(stdOut, stdErr, execResult.exitValue)
}
/**
* The [ExecutorService] being set in the given project.
* @throws IllegalStateException if there are no executor in the project.
*/
val Project.executor: ExecutorService
get() = this.convention.plugins["executor"] as? ExecutorService
?: throw IllegalStateException("Executor wasn't found")
/**
* Creates a new executor service with additional action [actionParameter] executed after the main one.
* The following is an example how to pass an environment parameter
* @code `executor.add(Action { it.environment = mapOf("JAVA_OPTS" to "-verbose:gc") })::execute`
*/
fun ExecutorService.add(actionParameter: Action<in ExecSpec>) = object : ExecutorService {
override fun execute(action: Action<in ExecSpec>): ExecResult? =
this@add.execute(Action {
action.execute(it)
actionParameter.execute(it)
})
}
/**
* Executes the [executable] with the given [arguments]
* and checks that the program finished with zero exit code.
*/
fun Project.executeAndCheck(executable: Path, arguments: List<String> = emptyList()) {
val (stdOut, stdErr, exitCode) = runProcess(
executor = executor::execute,
executable = executable.toString(),
args = arguments
)
println("""
|stdout: $stdOut
|stderr: $stdErr
""".trimMargin())
check(exitCode == 0) { "Execution failed with exit code: $exitCode" }
}
/**
* Returns [project]'s process executor.
* @see Project.exec
*/
fun localExecutor(project: Project) = { a: Action<in ExecSpec> -> project.exec(a) }
fun localExecutorService(project: Project): ExecutorService = object : ExecutorService {
override fun execute(action: Action<in ExecSpec>): ExecResult? = project.exec(action)
}
/**
* Executes a given action with iPhone Simulator.
*
* The test target should be specified with -Ptest_target=ios_x64
* @see KonanTarget.IOS_X64
* @param iosDevice an optional project property used to control simulator's device type
* Specify -PiosDevice=iPhone X to set it
*/
private fun simulator(project: Project): ExecutorService = object : ExecutorService {
private val target = project.testTarget
private val simctl by lazy {
val sdk = when (target) {
KonanTarget.TVOS_X64 -> Xcode.current.appletvsimulatorSdk
KonanTarget.IOS_X64 -> Xcode.current.iphonesimulatorSdk
KonanTarget.WATCHOS_X64,
KonanTarget.WATCHOS_X86 -> Xcode.current.watchsimulatorSdk
else -> error("Unexpected simulation target: $target")
}
val out = ByteArrayOutputStream()
val result = project.exec {
it.commandLine("/usr/bin/xcrun", "--find", "simctl", "--sdk", sdk)
it.standardOutput = out
}
result.assertNormalExitValue()
out.toString("UTF-8").trim()
}
private val device = project.findProperty("iosDevice")?.toString() ?: when (target) {
KonanTarget.TVOS_X64 -> "Apple TV 4K"
KonanTarget.IOS_X64 -> "iPhone 8"
KonanTarget.WATCHOS_X64,
KonanTarget.WATCHOS_X86 -> "Apple Watch Series 4 - 40mm"
else -> error("Unexpected simulation target: $target")
}
private val archSpecification = when (target.architecture) {
Architecture.X86 -> listOf("-a", "i386")
Architecture.X64 -> listOf() // x86-64 is used by default.
else -> error("${target.architecture} can't be used in simulator.")
}.toTypedArray()
override fun execute(action: Action<in ExecSpec>): ExecResult? = project.exec { execSpec ->
action.execute(execSpec)
// Starting Xcode 11 `simctl spawn` requires explicit `--standalone` flag.
with(execSpec) { commandLine = listOf(simctl, "spawn", "--standalone", *archSpecification, device, executable) + args }
}
}
/**
* Remote process executor.
*
* @param remote makes binaries be executed on a remote host
* Specify it as -Premote=user@host
*/
private fun sshExecutor(project: Project): ExecutorService = object : ExecutorService {
private val remote: String = project.property("remote").toString()
private val sshArgs: List<String> = System.getenv("SSH_ARGS")?.split(" ") ?: emptyList()
private val sshHome = System.getenv("SSH_HOME") ?: "/usr/bin"
// Unique remote dir name to be used in the target host
private val remoteDir = run {
val date = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss"))
Paths.get(project.findProperty("remoteRoot").toString(), "tmp",
System.getProperty("user.name") + "_" + date).toString()
}
override fun execute(action: Action<in ExecSpec>): ExecResult {
var execFile: String? = null
createRemoteDir()
val execResult = project.exec { execSpec ->
action.execute(execSpec)
with(execSpec) {
upload(executable)
executable = "$remoteDir/${File(executable).name}"
execFile = executable
commandLine = arrayListOf("$sshHome/ssh") + sshArgs + remote + commandLine
}
}
cleanup(execFile!!)
return execResult
}
private fun createRemoteDir() {
project.exec {
it.commandLine = arrayListOf("$sshHome/ssh") + sshArgs + remote + "mkdir" + "-p" + remoteDir
}
}
private fun upload(fileName: String) {
project.exec {
it.commandLine = arrayListOf("$sshHome/scp") + sshArgs + fileName + "$remote:$remoteDir"
}
}
private fun cleanup(fileName: String) {
project.exec {
it.commandLine = arrayListOf("$sshHome/ssh") + sshArgs + remote + "rm" + fileName
}
}
}
internal data class DeviceTarget(
@Expose val name: String,
@Expose val udid: String,
@Expose val state: String,
@Expose val type: String
)
private fun deviceLauncher(project: Project) = object : ExecutorService {
private val xcProject = Paths.get(project.testOutputRoot, "launcher")
private val idb = project.findProperty("idb_path") as? String ?: "idb"
private val deviceName = project.findProperty("device_name") as? String
override fun execute(action: Action<in ExecSpec>): ExecResult? {
var result: ExecResult? = null
try {
val udid = targetUDID()
println("Found device UDID: $udid")
install(udid, xcProject.resolve("build/KonanTestLauncher.ipa").toString())
val bundleId = "org.jetbrains.kotlin.KonanTestLauncher"
val commands = startDebugServer(udid, bundleId)
.split("\n")
.filter { it.isNotBlank() }
.flatMap { listOf("-o", it) }
var savedOut: OutputStream? = null
val out = ByteArrayOutputStream()
result = project.exec { execSpec: ExecSpec ->
action.execute(execSpec)
execSpec.executable = "lldb"
execSpec.args = commands + "-b" + "-o" + "command script import ${pythonScript()}" +
"-o" + ("process launch" +
(execSpec.args.takeUnless { it.isEmpty() }
?.let { " -- ${it.joinToString(" ")}" }
?: "")) +
"-o" + "get_exit_code" +
"-k" + "get_exit_code" +
"-k" + "exit -1"
// A test task that uses project.exec { } sets the stdOut to parse the result,
// but the test executable is being run under debugger that has its own output mixed with the
// output from the test. Save the stdOut from the test to write the parsed output to it.
savedOut = execSpec.standardOutput
execSpec.standardOutput = out
}
out.toString()
.also { if (project.verboseTest) println(it) }
.split("\n")
.dropWhile { s -> !s.startsWith("(lldb) process launch") }
.drop(1) // drop 'process launch' also
.dropLastWhile { !it.matches(".*Process [0-9]* exited with status .*".toRegex()) }
.joinToString("\n") {
it.replace("Process [0-9]* exited with status .*".toRegex(), "")
.replace("\r", "") // TODO: investigate: where does the \r comes from
}
.also {
savedOut?.write(it.toByteArray())
}
uninstall(udid, bundleId)
} catch (exc: Exception) {
throw RuntimeException("iOS-device execution failed", exc)
} finally {
kill()
}
return result
}
/*
* This script kills the target process in case it has been stopped,
* and exists lldb with the same exit code as a target process.
*/
private fun pythonScript(): String = xcProject.resolve("lldb_cmd.py").toFile().run {
writeText( // language=Python
"""
import lldb
def exit_code(debugger, command, exe_ctx, result, internal_dict):
process = exe_ctx.GetProcess()
state = process.GetState()
if state == lldb.eStateStopped:
# Call flush method, otherwise some output isn't shown in the debugger
debugger.HandleCommand("expression -- (int) fflush(NULL)")
debugger.HandleCommand("bt all")
process.Kill()
code = process.GetExitStatus()
debugger.HandleCommand("exit %d" % code)
def __lldb_init_module(debugger, _):
debugger.HandleCommand('command script add -f lldb_cmd.exit_code get_exit_code')
""".trimIndent())
absolutePath
}
private fun kill() = project.exec { it.commandLine(idb, "kill") }
private inline fun tryUntilTrue(times: Int = 3, f: () -> Boolean) {
for (i in 1..times) {
if (f()) break
else TimeUnit.SECONDS.sleep(i.toLong())
}
}
private fun targetUDID(): String {
val out = ByteArrayOutputStream()
// idb launches idb_companion but doesn't wait for it and just exits.
// So relaunch `list-targets` again.
tryUntilTrue {
project.exec {
it.commandLine(idb, "list-targets", "--json")
it.standardOutput = out
}.assertNormalExitValue()
out.toString().trim().isNotEmpty()
}
return out.toString().run {
check(isNotEmpty())
split("\n")
.filter { it.isNotEmpty() }
.map {
gson.fromJson(it, DeviceTarget::class.java)
}
.first {
it.type == "device" && deviceName?.run { this == it.name } ?: true
}
.udid
}
}
private fun install(udid: String, bundlePath: String) {
val out = ByteArrayOutputStream()
lateinit var result: ExecResult
tryUntilTrue {
result = project.exec {
it.workingDir = xcProject.toFile()
it.commandLine = listOf(idb, "install", "--udid", udid, bundlePath)
it.standardOutput = out
it.errorOutput = out
it.isIgnoreExitValue = true
}
println(out.toString())
result.exitValue == 0
}
check(result.exitValue == 0) { "Installation of $bundlePath failed: $out" }
}
private fun uninstall(udid: String, bundleId: String) {
val out = ByteArrayOutputStream()
project.exec {
it.workingDir = xcProject.toFile()
it.commandLine = listOf(idb, "uninstall", "--udid", udid, bundleId)
it.standardOutput = out
it.errorOutput = out
it.isIgnoreExitValue = true
}
println(out.toString())
}
private fun startDebugServer(udid: String, bundleId: String): String {
val out = ByteArrayOutputStream()
val result = project.exec {
it.workingDir = xcProject.toFile()
it.commandLine = listOf(idb, "debugserver", "start", "--udid", udid, bundleId)
it.standardOutput = out
it.errorOutput = out
it.isIgnoreExitValue = true
}
check(result.exitValue == 0) { "Failed to start debug server: $out" }
return out.toString()
}
}
fun KonanTestExecutable.configureXcodeBuild() {
this.doBeforeRun = Action {
val signIdentity = project.findProperty("sign_identity") as? String ?: "iPhone Developer"
val developmentTeam = project.findProperty("development_team") as? String
requireNotNull(developmentTeam) { "Specify '-Pdevelopment_team=' with the your team id" }
val xcProject = Paths.get(project.testOutputRoot, "launcher")
val shellScript: String = // language=Bash
mutableListOf("""
set -x
# Copy executable to the build dir.
COPY_TO="${"$"}TARGET_BUILD_DIR/${"$"}EXECUTABLE_PATH"
cp "${project.file(executable).absolutePath}" "${"$"}COPY_TO"
# copy dSYM if it exists
DSYM_DIR="${project.file("$executable.dSYM").absolutePath}"
if [ -d "${"$"}DSYM_DIR" ]; then
cp -r "${"$"}DSYM_DIR" "${"$"}TARGET_BUILD_DIR/${"$"}EXECUTABLE_FOLDER_PATH/"
fi
""".trimIndent()).also {
if (this is FrameworkTest) {
// Create a Frameworks folder inside the build dir.
it += "mkdir -p \"\$TARGET_BUILD_DIR/\$FRAMEWORKS_FOLDER_PATH\""
// Copy each framework to the Frameworks dir.
it += frameworks.filter { framework -> !framework.isStatic }
.map { framework ->
val artifact = framework.artifact
"cp -r \"$testOutput/${this.name}/${project.testTarget.name}/$artifact.framework\" " +
"\"\$TARGET_BUILD_DIR/\$FRAMEWORKS_FOLDER_PATH/$artifact.framework\""
}
}
}.joinToString(separator = "\\n") { it.replace("\"", "\\\"") }
// Copy template xcode project.
project.file("iosLauncher").copyRecursively(xcProject.toFile(), overwrite = true)
xcProject.resolve("KonanTestLauncher.xcodeproj/project.pbxproj")
.toFile()
.apply {
val text = readLines().joinToString("\n") {
when {
it.contains("CODE_SIGN_IDENTITY") ->
it.replaceAfter("= ", "\"$signIdentity\";")
it.contains("DEVELOPMENT_TEAM") || it.contains("DevelopmentTeam") ->
it.replaceAfter("= ", "$developmentTeam;")
it.contains("shellScript = ") ->
it.replaceAfter("= ", "\"$shellScript\";")
else -> it
}
}
writeText(text)
}
val sdk = when (project.testTarget) {
KonanTarget.IOS_ARM32, KonanTarget.IOS_ARM64 -> Xcode.current.iphoneosSdk
else -> error("Unsupported target: ${project.testTarget}")
}
fun xcodebuild(vararg elements: String) {
val xcode = listOf("/usr/bin/xcrun", "-sdk", sdk, "xcodebuild")
val out = ByteArrayOutputStream()
val result = project.exec {
it.workingDir = xcProject.toFile()
it.commandLine = xcode + elements.toList()
it.standardOutput = out
}
println(out.toString("UTF-8"))
result.assertNormalExitValue()
}
xcodebuild("-workspace", "KonanTestLauncher.xcodeproj/project.xcworkspace",
"-scheme", "KonanTestLauncher", "-allowProvisioningUpdates", "-destination",
"generic/platform=iOS", "build")
val archive = xcProject.resolve("build/KonanTestLauncher.xcarchive").toString()
xcodebuild("-workspace", "KonanTestLauncher.xcodeproj/project.xcworkspace",
"-scheme", "KonanTestLauncher", "archive", "-archivePath", archive)
xcodebuild("-exportArchive", "-archivePath", archive, "-exportOptionsPlist", "KonanTestLauncher/Info.plist",
"-exportPath", xcProject.resolve("build").toString())
}
}
@@ -0,0 +1,26 @@
package org.jetbrains.kotlin
import com.google.gson.annotations.*
import com.google.gson.*
import com.google.gson.stream.JsonReader
import java.io.File
import java.io.FileReader
import java.io.PrintWriter
data class ExternalTestReport(@Expose val statistics: Statistics, @Expose val groups: List<KonanTestGroupReport>)
fun saveReport(reportFileName: String, statistics: Statistics, groups:List<KonanTestGroupReport>){
File(reportFileName).apply {
parentFile.mkdirs()
PrintWriter(this).use {
it.append(gson.toJson(ExternalTestReport(statistics, groups)))
}
}
}
internal val gson = GsonBuilder().excludeFieldsWithoutExposeAnnotation().create()!!
fun loadReport(reportFileName: String) : ExternalTestReport = JsonReader(FileReader(reportFileName)).use {
gson.fromJson(it, ExternalTestReport::class.java)
}
@@ -0,0 +1,270 @@
package org.jetbrains.kotlin
import groovy.lang.Closure
import org.gradle.api.Action
import org.gradle.api.DefaultTask
import org.gradle.api.Task
import org.gradle.api.file.FileTree
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.TaskAction
import org.gradle.language.base.plugins.LifecycleBasePlugin
import org.jetbrains.kotlin.konan.target.*
import java.io.File
import java.io.FileWriter
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
/**
* Test task for -produce framework testing. Requires a framework to be built by the Konan plugin
* with konanArtifacts { framework(frameworkName, targets: [ testTarget] ) } and a dependency set
* according to a pattern "compileKonan${frameworkName}".
*
* @property swiftSources Swift-language test sources that use a given framework
* @property frameworks names of frameworks
*/
open class FrameworkTest : DefaultTask(), KonanTestExecutable {
@Input
lateinit var swiftSources: List<String>
@Input
lateinit var frameworks: MutableList<Framework>
@Input
var fullBitcode: Boolean = false
@Input
var codesign: Boolean = true
val testOutput: String = project.testOutputFramework
@Input @Optional
var expectedExitStatus: Int? = null
/**
* Framework description.
*
* @param name is the framework name,
* @param sources framework sources,
* @param bitcode bitcode embedding in the framework,
* @param isStatic determines that framework is static
* @param artifact the name of the resulting artifact,
* @param library library dependency name,
* @param opts additional options for the compiler.
*/
class Framework(
val name: String,
var sources: List<String> = emptyList(),
var bitcode: Boolean = false,
var isStatic: Boolean = false,
var artifact: String = name,
var library: String? = null,
var opts: List<String> = emptyList()
)
/**
* Used for the framework configuration in the task's closure.
*/
fun framework(name: String, closure: Closure<Framework>): Framework {
val f = Framework(name).apply {
closure.delegate = this
closure.resolveStrategy = Closure.DELEGATE_FIRST
closure.call()
// map to file paths
sources = sources.toFiles(Language.Kotlin).map { it.path }
}
if (!::frameworks.isInitialized) {
frameworks = mutableListOf(f)
} else {
frameworks.add(f)
}
return f
}
enum class Language(val extension: String) {
Kotlin(".kt"), ObjC(".m"), Swift(".swift")
}
fun Language.filesFrom(dir: String): FileTree = project.fileTree(dir) {
// include only files with the language extension
it.include("*${this.extension}")
}
fun List<String>.toFiles(language: Language): List<File> =
this.map { language.filesFrom(it) }
.flatMap { it.files }
override val executable: String
get() = Paths.get(testOutput, name, "swiftTestExecutable").toString()
override var doBeforeRun: Action<in Task>? = null
override var doBeforeBuild: Action<in Task>? = null
override val buildTasks: List<Task>
get() = frameworks.map { project.tasks.getByName("compileKonan${it.name}") }
@Suppress("UnstableApiUsage")
override fun configure(config: Closure<*>): Task {
super.configure(config)
// set crossdist build dependency if custom konan.home wasn't set
this.dependsOnDist()
// Set Gradle properties for the better navigation
group = LifecycleBasePlugin.VERIFICATION_GROUP
description = "Kotlin/Native test infrastructure task"
check(::frameworks.isInitialized) { "Frameworks should be set" }
return this
}
private fun buildTestExecutable() {
val frameworkParentDirPath = "$testOutput/$name/${project.testTarget.name}"
frameworks.forEach { framework ->
val frameworkArtifact = framework.artifact
val frameworkPath = "$frameworkParentDirPath/$frameworkArtifact.framework"
val frameworkBinaryPath = "$frameworkPath/$frameworkArtifact"
validateBitcodeEmbedding(frameworkBinaryPath)
if (codesign) codesign(project, frameworkPath)
}
// create a test provider and get main entry point
val provider = Paths.get(testOutput, name, "provider.swift")
FileWriter(provider.toFile()).use { writer ->
val providers = swiftSources.toFiles(Language.Swift)
.map { it.name.toString().removeSuffix(".swift").capitalize() }
.map { "${it}Tests" }
writer.write("""
|// THIS IS AUTOGENERATED FILE
|// This method is invoked by the main routine to get a list of tests
|func registerProviders() {
| ${providers.joinToString("\n ") { "$it()" }}
|}
""".trimMargin())
}
val testHome = project.file("framework").toPath()
val swiftMain = Paths.get(testHome.toString(), "main.swift").toString()
// Compile swift sources
val sources = swiftSources.toFiles(Language.Swift)
.map { it.path } + listOf(provider.toString(), swiftMain)
val options = listOf(
"-g",
"-Xlinker", "-rpath", "-Xlinker", "@executable_path/Frameworks",
"-Xlinker", "-rpath", "-Xlinker", frameworkParentDirPath,
"-F", frameworkParentDirPath,
"-Xcc", "-Werror" // To fail compilation on warnings in framework header.
)
compileSwift(project, project.testTarget, sources, options, Paths.get(executable), fullBitcode)
}
@TaskAction
fun run() {
// Build test executable as a first action of the task before executing the test
buildTestExecutable()
doBeforeRun?.execute(this)
runTest(executorService = project.executor, testExecutable = Paths.get(executable))
}
/**
* Returns path to directory that contains `libswiftCore.dylib` for the current
* test target.
*/
private fun getSwiftLibsPathForTestTarget(): String {
val target = project.testTarget
val platform = project.platformManager.platform(target)
val configs = platform.configurables as AppleConfigurables
val swiftPlatform = when (target) {
KonanTarget.IOS_X64 -> "iphonesimulator"
KonanTarget.IOS_ARM32, KonanTarget.IOS_ARM64 -> "iphoneos"
KonanTarget.TVOS_X64 -> "appletvsimulator"
KonanTarget.TVOS_ARM64 -> "appletvos"
KonanTarget.MACOS_X64 -> "macosx"
KonanTarget.WATCHOS_ARM64 -> "watchos"
KonanTarget.WATCHOS_X64, KonanTarget.WATCHOS_X86 -> "watchsimulator"
else -> throw IllegalStateException("Test target $target is not supported")
}
val simulatorPath = when (target) {
KonanTarget.TVOS_X64,
KonanTarget.IOS_X64,
KonanTarget.WATCHOS_X86,
KonanTarget.WATCHOS_X64 -> Xcode.current.getLatestSimulatorRuntimeFor(target, configs.osVersionMin)?.bundlePath?.let {
"$it/Contents/Resources/RuntimeRoot/usr/lib/swift"
}
else -> null
}
// Use default path from toolchain if we cannot get `bundlePath` for target.
// It may be the case for simulators if Xcode/macOS is old.
return simulatorPath ?: configs.absoluteTargetToolchain + "/usr/lib/swift-5.0/$swiftPlatform"
}
private fun buildEnvironment(): Map<String, String> {
val target = project.testTarget
// Hopefully, lexicographical comparison will work.
val newMacos = System.getProperty("os.version").compareTo("10.14.4") >= 0
val dyldLibraryPathKey = when (target) {
KonanTarget.IOS_X64,
KonanTarget.WATCHOS_X86,
KonanTarget.WATCHOS_X64,
KonanTarget.TVOS_X64 -> "SIMCTL_CHILD_DYLD_LIBRARY_PATH"
else -> "DYLD_LIBRARY_PATH"
}
return if (newMacos && target == KonanTarget.MACOS_X64) emptyMap() else mapOf(
dyldLibraryPathKey to getSwiftLibsPathForTestTarget()
)
}
private fun runTest(executorService: ExecutorService, testExecutable: Path, args: List<String> = emptyList()) {
val (stdOut, stdErr, exitCode) = runProcess(
executor = { executorService.add(Action {
it.environment = buildEnvironment()
it.workingDir = Paths.get(testOutput).toFile()
}).execute(it) },
executable = testExecutable.toString(),
args = args)
val testExecName = testExecutable.fileName
println("""
|$testExecName
|stdout: $stdOut
|stderr: $stdErr
""".trimMargin())
check(exitCode == expectedExitStatus ?: 0) { "Execution of $testExecName failed with exit code: $exitCode " }
}
private fun validateBitcodeEmbedding(frameworkBinary: String) {
// Check only the full bitcode embedding for now.
if (!fullBitcode) {
return
}
val testTarget = project.testTarget
val configurables = project.platformManager.platform(testTarget).configurables as AppleConfigurables
val bitcodeBuildTool = "${configurables.absoluteAdditionalToolsDir}/bin/bitcode-build-tool"
val toolPath = "${configurables.absoluteTargetToolchain}/usr/bin/"
val sdk = when (testTarget) {
KonanTarget.IOS_X64,
KonanTarget.TVOS_X64,
KonanTarget.WATCHOS_X86,
KonanTarget.WATCHOS_X64 -> return // bitcode-build-tool doesn't support simulators.
KonanTarget.IOS_ARM64,
KonanTarget.IOS_ARM32 -> Xcode.current.iphoneosSdk
KonanTarget.MACOS_X64 -> Xcode.current.macosxSdk
KonanTarget.TVOS_ARM64 -> Xcode.current.appletvosSdk
KonanTarget.WATCHOS_ARM32,
KonanTarget.WATCHOS_ARM64 -> Xcode.current.watchosSdk
else -> error("Cannot validate bitcode for test target $testTarget")
}
val python3 = listOf("/usr/bin/python3", "/usr/local/bin/python3")
.map { Paths.get(it) }.firstOrNull { Files.exists(it) }
?: error("Can't find python3")
runTest(executorService = localExecutorService(project), testExecutable = python3,
args = listOf("-B", bitcodeBuildTool, "--sdk", sdk, "-v", "-t", toolPath, frameworkBinary))
}
}
@@ -0,0 +1,48 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import org.gradle.api.NamedDomainObjectCollection
import org.gradle.api.NamedDomainObjectContainer
import org.gradle.api.Project
import org.gradle.api.file.FileCollection
import org.gradle.api.plugins.ExtraPropertiesExtension
import org.gradle.api.provider.Provider
import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
import org.jetbrains.kotlin.gradle.plugin.KotlinCompilation
import org.jetbrains.kotlin.gradle.plugin.KotlinTargetPreset
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeCompilation
import java.io.File
/*
* This file includes internal short-cuts visible only inside of the 'buildSrc' module.
*/
internal val hostOs by lazy { System.getProperty("os.name") }
internal val userHome by lazy { System.getProperty("user.home") }
internal val Project.ext: ExtraPropertiesExtension
get() = extensions.getByName("ext") as ExtraPropertiesExtension
internal val Project.kotlin: KotlinMultiplatformExtension
get() = extensions.getByName("kotlin") as KotlinMultiplatformExtension
internal val NamedDomainObjectCollection<KotlinTargetPreset<*>>.macosX64: KotlinTargetPreset<*>
get() = getByName(::macosX64.name) as KotlinTargetPreset<*>
internal val NamedDomainObjectCollection<KotlinTargetPreset<*>>.linuxX64: KotlinTargetPreset<*>
get() = getByName(::linuxX64.name) as KotlinTargetPreset<*>
internal val NamedDomainObjectCollection<KotlinTargetPreset<*>>.mingwX64: KotlinTargetPreset<*>
get() = getByName(::mingwX64.name) as KotlinTargetPreset<*>
internal val NamedDomainObjectContainer<out KotlinCompilation<*>>.main: KotlinNativeCompilation
get() = getByName(::main.name) as KotlinNativeCompilation
internal val FileCollection.isNotEmpty: Boolean
get() = !isEmpty
internal fun Provider<File>.resolve(child: String): Provider<File> = map { it.resolve(child) }
@@ -0,0 +1,47 @@
package org.jetbrains.kotlin
import groovy.lang.Closure
import org.gradle.api.Task
import org.gradle.api.tasks.Exec
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.InputFile
import org.gradle.api.tasks.OutputDirectory
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.target.HostManager
import java.io.File
// TODO: Implement as a part of the gradle plugin
open class KlibInstall: Exec() {
@InputFile
lateinit var klib: File
var repo: File = project.rootDir
val installDir: File
@OutputDirectory
get() {
val klibName = klib.name.take(klib.name.lastIndexOf('.'))
return project.file("${repo.absolutePath}/$klibName")
}
@Input
var target: String = HostManager.hostName
override fun configure(config: Closure<*>): Task {
val result = super.configure(config)
val konanHomePath = project.findProperty("org.jetbrains.kotlin.native.home") ?: "dist"
val konanHome = project.rootProject.file(konanHomePath)
val suffix = if (HostManager.host == KonanTarget.MINGW_X64) ".bat" else ""
val klibProgram = "$konanHome/bin/klib$suffix"
doFirst {
repo.mkdirs()
commandLine(klibProgram,
"install", klib.absolutePath,
"-target", target,
"-repository", repo.absolutePath
)
}
return result
}
}
@@ -0,0 +1,33 @@
package org.jetbrains.kotlin
import org.gradle.api.Action
import org.gradle.api.Task
/**
* An interface that any test that works with ExecutorService
* should implement to be run on different platforms.
*/
interface KonanTestExecutable : Task {
/**
* Test executable to be run by the service.
*/
val executable: String
/**
* Action that configures task or does some workload before the test will be executed.
* Could be done as a first step in the test or just as a `doFirst` action in the test task.
*/
var doBeforeRun: Action<in Task>?
/**
* Action that configures task or does some workload before the test will be built.
* Depending on the test task implementation this action is done before the build task
* or as its `doFirst` action.
*/
var doBeforeBuild: Action<in Task>?
/**
* Build tasks that this [executable] depends on, or is built from.
*/
val buildTasks: List<Task>
}
@@ -0,0 +1,164 @@
package org.jetbrains.kotlin
import com.google.gson.annotations.Expose
import java.io.PrintWriter
import java.io.StringWriter
import org.gradle.api.Project
enum class TestStatus {
PASSED,
FAILED,
ERROR,
SKIPPED
}
data class Statistics(
@Expose var passed: Int = 0,
@Expose var failed: Int = 0,
@Expose var error: Int = 0,
@Expose var skipped: Int = 0) {
fun pass(count: Int = 1) { passed += count }
fun skip(count: Int = 1) { skipped += count }
fun fail(count: Int = 1) { failed += count }
fun error(count: Int = 1) { error += count }
fun add(other: Statistics) {
passed += other.passed
failed += other.failed
error += other.error
skipped += other.skipped
}
}
val Statistics.total: Int
get() = passed + failed + error + skipped
class TestFailedException(msg:String) : RuntimeException(msg)
data class KonanTestGroupReport(@Expose val name: String, val suites: List<KonanTestSuiteReport>)
data class KonanTestSuiteReport(@Expose val name: String, val tests: List<KonanTestCaseReport>)
data class KonanTestCaseReport(@Expose val name: String, @Expose val status: TestStatus, @Expose val comment: String? = null)
class KonanTestSuiteReportEnvironment(val name: String, val project: Project, val statistics: Statistics) {
private val tc = if (Tc.enabled) TeamCityTestPrinter(project) else null
val tests = mutableListOf<KonanTestCaseReport>()
fun executeTest(testName: String, action:() -> Unit) {
var test: KonanTestCaseReport?
try {
tc?.startTest(testName)
action()
tc?.passTest(testName)
statistics.pass()
test = KonanTestCaseReport(testName, TestStatus.PASSED)
} catch (e:TestFailedException) {
tc?.failedTest(testName, e)
statistics.fail()
test = KonanTestCaseReport(testName, TestStatus.FAILED, "Exception: ${e.message}. Cause: ${e.cause?.message}")
project.logger.quiet("test: $testName failed")
} catch (e:Exception) {
tc?.errorTest(testName, e)
statistics.error()
test = KonanTestCaseReport(testName, TestStatus.ERROR, "Exception: ${e.message}. Cause: ${e.cause?.message}")
project.logger.quiet("error on test: $testName", e)
}
test!!.apply {
tests += this
if (status == TestStatus.ERROR || status == TestStatus.FAILED) {
project.logger.quiet("Command to reproduce: ./gradlew $name -Pfilter=${test.name}\n")
}
}
}
fun skipTest(name: String) {
tc?.skipTest(name)
statistics.skip()
tests += KonanTestCaseReport(name, TestStatus.SKIPPED)
}
fun abort(throwable: Throwable, count:Int) {
statistics.error(count)
project.logger.quiet("suite `$name` aborted with exception", throwable)
}
internal operator fun invoke(action: (KonanTestSuiteReportEnvironment) -> Unit) {
tc?.suiteStart(name)
action(this)
tc?.suiteFinish(name)
}
}
class KonanTestGroupReportEnvironment(val project:Project) {
val statistics = Statistics()
val suiteReports = mutableListOf<KonanTestSuiteReport>()
fun suite(suiteName:String, action:(KonanTestSuiteReportEnvironment)->Unit) {
val konanTestSuiteEnvironment = KonanTestSuiteReportEnvironment(suiteName, project, statistics)
konanTestSuiteEnvironment {
action(it)
}
suiteReports += KonanTestSuiteReport(suiteName, konanTestSuiteEnvironment.tests)
}
}
private class TeamCityTestPrinter(val project:Project) {
fun suiteStart(name: String) {
teamcityReport("testSuiteStarted name='$name'")
}
fun suiteFinish(name: String) {
teamcityReport("testSuiteFinished name='$name'")
}
fun startTest(name: String) {
teamcityReport("testStarted name='$name'")
}
fun passTest(testName: String) = teamcityFinish(testName)
fun failedTest(testName: String, testFailedException: TestFailedException) {
teamcityReport("testFailed type='comparisonFailure' name='$testName' message='${testFailedException.message.toTeamCityFormat()}'")
teamcityFinish(testName)
}
fun errorTest(testName: String, exception: Exception) {
val writer = StringWriter()
exception.printStackTrace(PrintWriter(writer))
val rawString = writer.toString()
teamcityReport("testFailed name='$testName' message='${exception.message.toTeamCityFormat()}' " +
"details='${rawString.toTeamCityFormat()}'")
teamcityFinish(testName)
}
fun skipTest(testName: String) {
teamcityReport("testIgnored name='$testName'")
teamcityFinish(testName)
}
private fun teamcityFinish(testName:String) {
teamcityReport("testFinished name='$testName'")
}
/**
* Teamcity require escaping some symbols in pipe manner.
* https://github.com/GitTools/GitVersion/issues/94
*/
private fun String?.toTeamCityFormat(): String = this?.let {
it.replace("\\|", "||")
.replace("\r", "|r")
.replace("\n", "|n")
.replace("'", "|'")
.replace("\\[", "|[")
.replace("]", "|]")} ?: "null"
private fun teamcityReport(msg: String) {
project.logger.quiet("##teamcity[$msg]")
}
}
@@ -0,0 +1,106 @@
package org.jetbrains.kotlin
import io.ktor.client.HttpClient
import io.ktor.client.request.get
import io.ktor.client.request.header
import io.ktor.client.request.post
import io.ktor.content.TextContent
import io.ktor.http.ContentType
import kotlinx.coroutines.runBlocking
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.options.Option
@Suppress("UnstableApiUsage")
open class KotlinBuildPusher : DefaultTask() {
@Input
@Option(option = "token", description = "Teamcity Bear token")
var token: String? = ""
@Input
@Option(option = "buildServer", description = "Teamcity server")
var buildServer: String = ""
@Input
@Option(option = "compilerConfigurationId", description = "Teamcity configuration id")
var compilerConfigurationId: String = "Kotlin_dev_Compiler"
@Input
@Option(option = "overrideConfigurationId", description = "Teamcity kotlin override configuration id")
var overrideConfigurationId: String = "Kotlin_dev_DeployMavenArtifacts_OverrideNative"
@Input
@Option(option = "kotlinVersion", description = "Kotlin Compiler Version")
var kotlinVersion: String = ""
@Input
@Option(option = "konanVersion", description = "Kotlin Native Compiler Version")
var konanVersion: String = ""
@TaskAction
fun run() {
requireNotNull(token, { "Teamcity Bear token required" })
val client = HttpClient()
runBlocking {
val buildId = client.get<String>(
scheme = "https",
host = buildServer,
path = "/app/rest/builds/buildType:(id:$compilerConfigurationId),number:$kotlinVersion/id"
) {
header("Authorization", "Bearer $token")
}
project.logger.info("pusher: buildId: $buildId")
/**
* <build>
* <buildType id="Kotlin_dev_DeployMavenArtifacts_OverrideNative"/>
* <lastChanges>
* <change locator="build:2330798"/>
* </lastChanges>
* <properties>
* <property name="system.versions.kotlin-native" value="1.3.50-dev-10483"/>
* </properties>
* </build>
*/
val content = buildString {
build {
buildType(overrideConfigurationId)
lastChanges {
change(buildId)
}
properties {
property("system.versions.kotlin-native", konanVersion)
}
}
}
project.logger.info("pusher: content: \"$content\"")
val res = client.post<String>(
scheme = "https",
host = buildServer,
path = "/app/rest/buildQueue"
) {
header("Authorization", "Bearer $token")
header("Origin", "https://$buildServer")
body = TextContent(content, ContentType.Application.Xml)
}
project.logger.info("pusher: result: \"$res\"")
}
client.close()
}
}
internal fun StringBuilder.paired(tag:String, body:StringBuilder.()->Unit) {
appendln("<$tag>")
body()
appendln("</$tag>")
}
internal fun StringBuilder.build(body: StringBuilder.() -> Unit) = paired("build", body)
internal fun StringBuilder.buildType(id: String) = appendln("<buildType id=\"$id\"/>")
internal fun StringBuilder.lastChanges(body: StringBuilder.() -> Unit) = paired("lastChanges", body)
internal fun StringBuilder.change(build: String) = appendln("<change locator=\"build:$build\"/>")
internal fun StringBuilder.properties(body: StringBuilder.() -> Unit) = paired("properties", body)
internal fun StringBuilder.property(key: String, value: String) = appendln("<property name=\"$key\" value=\"$value\"/>")
@@ -0,0 +1,463 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin
import groovy.lang.Closure
import org.gradle.api.Action
import org.gradle.api.DefaultTask
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.provider.Provider
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.TaskAction
import org.gradle.language.base.plugins.LifecycleBasePlugin
import org.gradle.process.ExecSpec
import java.io.File
import java.io.ByteArrayOutputStream
import java.util.regex.Pattern
import org.jetbrains.kotlin.konan.target.HostManager
abstract class KonanTest : DefaultTask(), KonanTestExecutable {
enum class Logger {
EMPTY, // Built without test runner
GTEST, // Google test log output
TEAMCITY, // TeamCity log output
SIMPLE, // Prints simple messages of passed/failed tests
SILENT // Prints no log of passed/failed tests
}
var disabled: Boolean
get() = !enabled
set(value) { enabled = !value }
/**
* Test output directory. Used to store processed sources and binary artifacts.
*/
abstract val outputDirectory: String
/**
* Test logger to be used for the test built with TestRunner (`-tr` option).
*/
abstract var testLogger: Logger
/**
* Test executable arguments.
*/
@Input
var arguments = mutableListOf<String>()
/**
* Test executable.
*/
abstract override val executable: String
/**
* Test source.
*/
lateinit var source: String
/**
* Sets test filtering to choose the exact test in the executable built with TestRunner.
*/
@Input
var useFilter = true
/**
* An action to be executed before the build.
* As this run task comes after the build task all actions for doFirst
* should be done before the build and not run.
*/
@Input @Optional
override var doBeforeBuild: Action<in Task>? = null
@Input @Optional
override var doBeforeRun: Action<in Task>? = null
override val buildTasks: List<Task>
get() = listOf(project.findKonanBuildTask(name, project.testTarget))
@Suppress("UnstableApiUsage")
override fun configure(config: Closure<*>): Task {
super.configure(config)
// Set Gradle properties for the better navigation
group = LifecycleBasePlugin.VERIFICATION_GROUP
description = "Kotlin/Native test infrastructure task"
if (testLogger != Logger.EMPTY) {
arguments.add("--ktest_logger=$testLogger")
}
if (useFilter && ::source.isInitialized) {
arguments.add("--ktest_filter=${source.convertToPattern()}")
}
this.dependsOnDist()
return this
}
@TaskAction
open fun run() = project.executeAndCheck(project.file(executable).toPath(), arguments)
// Converts to runner's pattern
private fun String.convertToPattern() = this.removeSuffix(".kt").replace("/", ".") + ".*"
internal fun ProcessOutput.print(prepend: String = "") {
if (project.verboseTest)
println(prepend + """
|stdout:
|$stdOut
|stderr:
|$stdErr
|exit code: $exitCode
""".trimMargin())
}
}
/**
* Create a test task of the given type. Supports configuration with Closure passed form build.gradle file.
*/
fun <T: KonanTestExecutable> Project.createTest(name: String, type: Class<T>, config: Closure<*>): T =
project.tasks.create(name, type).apply {
// Apply closure set in build.gradle to get all parameters.
this.configure(config)
if (enabled) {
// If run task depends on something, build tasks should also depend on this.
buildTasks.forEach { buildTask ->
buildTask.sameDependenciesAs(this)
// Run task should depend on compile task
this.dependsOn(buildTask)
doBeforeBuild?.let { buildTask.doFirst(it) }
buildTask.enabled = enabled
}
}
}
/**
* Task to run tests compiled with TestRunner.
* Runs tests with GTEST output and parses it to create statistics info
*/
open class KonanGTest : KonanTest() {
override val outputDirectory = "${project.testOutputStdlib}/$name"
// Use GTEST logger to parse test results later
override var testLogger = Logger.GTEST
override val executable: String
get() = "$outputDirectory/${project.testTarget.name}/$name.${project.testTarget.family.exeSuffix}"
var statistics = Statistics()
@TaskAction
override fun run() {
doBeforeRun?.execute(this)
runProcess(
executor = {project.executor.execute(it)},
executable = executable,
args = arguments
).run {
parse(stdOut)
println("""
|stdout:
|$stdOut
|stderr:
|$stdErr
|exit code: $exitCode
""".trimMargin())
check(exitCode == 0) { "Test $executable exited with $exitCode" }
}
}
private fun parse(output: String) = statistics.apply {
Pattern.compile("\\[ PASSED ] ([0-9]*) tests\\.").matcher(output)
.apply { if (find()) pass(group(1).toInt()) }
Pattern.compile("\\[ FAILED ] ([0-9]*) tests.*").matcher(output)
.apply { if (find()) fail(group(1).toInt()) }
if (total == 0) {
// No test were run. Try to find if we've tried to run something
error(Pattern.compile("\\[={10}] Running ([0-9]*) tests from ([0-9]*) test cases\\..*")
.matcher(output)
.run { if (find()) group(1).toInt() else 1 })
}
}
}
/**
* Task to run tests built into a single predefined binary named `localTest`.
* Note: this task should depend on task that builds a test binary.
*/
open class KonanLocalTest : KonanTest() {
override val outputDirectory = project.testOutputLocal
// local tests built into a single binary with the known name
override val executable: String
get() = "$outputDirectory/${project.testTarget.name}/localTest.${project.testTarget.family.exeSuffix}"
override var testLogger = Logger.SILENT
@Input @Optional
var expectedExitStatus: Int? = null
@Input @Optional
var expectedExitStatusChecker: (Int) -> Boolean = { it == (expectedExitStatus ?: 0) }
/**
* Should this test fail or not.
*/
@Input @Optional
var expectedFail = false
/**
* Used to validate output as a gold value.
*/
@Input @Optional
var goldValue: String? = null
/**
* Checks test's output against gold value and returns true if the output matches the expectation.
*/
@Input @Optional
var outputChecker: (String) -> Boolean = { str -> goldValue == null || goldValue == str }
/**
* Input test data to be passed to process' stdin.
*/
@Input @Optional
var testData: String? = null
/**
* Should compiler message be read and validated with output checker or gold value.
*/
@Input @Optional
var compilerMessages = false
@Input @Optional
var multiRuns = false
@Input @Optional
var multiArguments: List<List<String>>? = null
@TaskAction
override fun run() {
doBeforeRun?.execute(this)
val times = if (multiRuns && multiArguments != null) multiArguments!!.size else 1
var output = ProcessOutput("", "", 0)
for (i in 1..times) {
val args = arguments + (multiArguments?.get(i - 1) ?: emptyList())
output += if (testData != null)
runProcessWithInput({project.executor.execute(it)}, executable, args, testData!!)
else
runProcess({project.executor.execute(it)}, executable, args)
}
if (compilerMessages) {
// TODO: as for now it captures output only in the driver task.
// It should capture output from the build task using Gradle's LoggerManager and LoggerOutput
val compilationLog = project.file("$executable.compilation.log").readText()
// TODO: ugly hack to fix irrelevant warnings.
val filteredCompilationLog = compilationLog.split('\n').filter {
it != "warning: relaxed memory model is not yet fully functional"
}.joinToString(separator = "\n")
output.stdOut = filteredCompilationLog + output.stdOut
}
output.check()
output.print()
}
private operator fun ProcessOutput.plus(other: ProcessOutput) = ProcessOutput(
stdOut + other.stdOut,
stdErr + other.stdErr,
exitCode + other.exitCode)
private fun ProcessOutput.check() {
val exitCodeMismatch = !expectedExitStatusChecker(exitCode)
if (exitCodeMismatch) {
val message = if (expectedExitStatus != null)
"Expected exit status: $expectedExitStatus, actual: $exitCode"
else
"Actual exit status doesn't match with exit status checker: $exitCode"
check(expectedFail) { """
|Test failed. $message
|stdout:
|$stdOut
|stderr:
|$stdErr
""".trimMargin()
}
println("Expected failure. $message")
}
val result = stdOut + stdErr
val goldValueMismatch = !outputChecker(result.replace(System.lineSeparator(), "\n"))
if (goldValueMismatch) {
val message = if (goldValue != null)
"Expected output: $goldValue, actual output: $result"
else
"Actual output doesn't match with output checker: $result"
check(expectedFail) { "Test failed. $message" }
println("Expected failure. $message")
}
check ((exitCodeMismatch || goldValueMismatch) || !expectedFail) { """
|Unexpected pass:
| * exit code mismatch: $exitCodeMismatch
| * gold value mismatch: $goldValueMismatch
| * expected fail: $expectedFail
""".trimMargin()
}
}
}
/**
* Executes a standalone tests provided with either @param executable or by the tasks @param name.
* The executable itself should be built by the konan plugin.
*/
open class KonanStandaloneTest : KonanLocalTest() {
init {
useFilter = false
}
override val outputDirectory: String
get() = "${project.testOutputLocal}/$name"
override var testLogger = Logger.EMPTY
override val executable: String
get() = "$outputDirectory/${project.testTarget.name}/$name.${project.testTarget.family.exeSuffix}"
@Input @Optional
var enableKonanAssertions = true
/**
* Compiler flags used to build a test.
*/
var flags: List<String> = listOf()
get() = if (enableKonanAssertions) field + "-ea" else field
fun getSources(): Provider<List<String>> = project.provider {
val sources = buildCompileList(project.file(source).toPath(), outputDirectory)
sources.forEach { it.writeTextToFile() }
sources.map { it.path }
}
}
/**
* This is another way to run the konanc compiler. It runs a konanc shell script.
*
* @note This task is not intended for regular testing as project.exec + a shell script isolate the jvm from IDEA.
* @see KonanLocalTest to be used as a regular task.
*/
open class KonanDriverTest : KonanStandaloneTest() {
override fun configure(config: Closure<*>): Task {
super.configure(config)
doFirst { konan() }
doBeforeBuild?.let { doFirst(it) }
return this
}
private fun konan() {
val dist = project.kotlinNativeDist
val konancDriver = if (HostManager.hostIsMingw) "konanc.bat" else "konanc"
val konanc = File("${dist.canonicalPath}/bin/$konancDriver").absolutePath
File(executable).parentFile.mkdirs()
val args = mutableListOf("-output", executable).apply {
if (project.testTarget != HostManager.host) {
add("-target")
add(project.testTarget.visibleName)
}
addAll(getSources().get())
addAll(flags)
addAll(project.globalTestArgs)
}
// run konanc compiler locally
runProcess(localExecutor(project), konanc, args).let {
it.print("Konanc compiler execution:")
project.file("$executable.compilation.log").run {
writeText(it.stdOut)
writeText(it.stdErr)
}
check(it.exitCode == 0) { "Compiler failed with exit code ${it.exitCode}" }
}
}
}
open class KonanInteropTest : KonanStandaloneTest() {
/**
* Name of the interop library
*/
@Input
lateinit var interop: String
}
open class KonanLinkTest : KonanStandaloneTest() {
@Input
lateinit var lib: String
}
/**
* Test task to check a library built by `-produce dynamic`.
* C source code should contain `testlib` as a reference to a testing library.
* It will be replaced then by the actual library name.
*/
open class KonanDynamicTest : KonanStandaloneTest() {
override fun configure(config: Closure<*>): Task {
super.configure(config)
doFirst { clang() }
return this
}
/**
* File path to the C source.
*/
@Input
lateinit var cSource: String
var clangTool = "clang"
// Replace testlib_api.h and all occurrences of the testlib with the actual name of the test
private fun processCSource(): String {
val sourceFile = File(cSource)
val prefixedName = if (HostManager.hostIsMingw) name else "lib$name"
val res = sourceFile.readText()
.replace("#include \"testlib_api.h\"", "#include \"${prefixedName}_api.h\"")
.replace("testlib", prefixedName)
val newFileName = "$outputDirectory/${sourceFile.name}"
println(newFileName)
File(newFileName).run {
createNewFile()
writeText(res)
}
return newFileName
}
private fun clang() {
val log = ByteArrayOutputStream()
val plugin = project.convention.getPlugin(ExecClang::class.java)
val execResult = plugin.execKonanClang(project.testTarget, Action<ExecSpec> {
it.workingDir = File(outputDirectory)
it.executable = clangTool
val artifactsDir = "$outputDirectory/${project.testTarget}"
it.args = listOf(processCSource(),
"-o", executable,
"-I", artifactsDir,
"-L", artifactsDir,
"-l", name,
"-Wl,-rpath,$artifactsDir")
it.standardOutput = log
it.errorOutput = log
it.isIgnoreExitValue = true
})
log.toString("UTF-8").also {
project.file("$executable.compilation.log").writeText(it)
println(it)
}
execResult.assertNormalExitValue()
}
}
@@ -0,0 +1,54 @@
package org.jetbrains.kotlin
import com.google.gson.*
import com.google.gson.annotations.*
data class LlvmCovReportFunction(
@Expose val name: String,
@Expose val count: Int,
@Expose val regions: List<List<Int>>,
@Expose val filenames: List<String>
)
data class LlvmCovReportSummary(
@Expose val lines: LlvmCovReportStatistics,
@Expose val functions: LlvmCovReportStatistics,
@Expose val instantiations: LlvmCovReportStatistics,
@Expose val regions: LlvmCovReportStatistics
)
/**
* TODO: Add support for `segments` field later.
* It's a bit complicated since every segment
* is encoded not as dictionary, but as array of ints and bools.
*/
data class LlvmCovReportFile(
@Expose val filename: String,
@Expose val summary: LlvmCovReportSummary
)
data class LlvmCovReportStatistics(
@Expose val count: Int,
@Expose val covered: Int,
@Expose val percent: Double
)
data class LlvmCovReportData(
@Expose val files: List<LlvmCovReportFile>,
@Expose val functions: List<LlvmCovReportFunction>,
@Expose val totals: LlvmCovReportSummary
)
data class LlvmCovReport(
@Expose val version: String,
@Expose val type: String,
@Expose val data: List<LlvmCovReportData>
)
fun parseLlvmCovReport(llvmCovReport: String): LlvmCovReport = gson.fromJson(llvmCovReport, LlvmCovReport::class.java)
val LlvmCovReport.isValid
get() = type == "llvm.coverage.json.export"
@@ -0,0 +1,268 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
@file:JvmName("MPPTools")
package org.jetbrains.kotlin
import groovy.lang.Closure
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.tasks.TaskState
import org.gradle.api.execution.TaskExecutionListener
import org.jetbrains.kotlin.gradle.plugin.KotlinTargetPreset
import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType
import org.jetbrains.kotlin.gradle.tasks.KotlinNativeLink
import org.jetbrains.kotlin.konan.target.HostManager
import org.jetbrains.report.*
import org.jetbrains.report.json.*
import java.nio.file.Paths
import java.io.File
import java.io.FileInputStream
import java.io.BufferedOutputStream
import java.io.BufferedInputStream
import java.net.HttpURLConnection
import java.net.URL
import java.util.Base64
/*
* This file includes short-cuts that may potentially be implemented in Kotlin MPP Gradle plugin in the future.
*/
// Short-cuts for mostly used paths.
@get:JvmName("mingwPath")
val mingwPath by lazy { System.getenv("MINGW64_DIR") ?: "c:/msys64/mingw64" }
@get:JvmName("kotlinNativeDataPath")
val kotlinNativeDataPath by lazy {
System.getenv("KONAN_DATA_DIR") ?: Paths.get(userHome, ".konan").toString()
}
// A short-cut for evaluation of the default host Kotlin/Native preset.
@JvmOverloads
fun defaultHostPreset(
subproject: Project,
whitelist: List<KotlinTargetPreset<*>> = listOf(subproject.kotlin.presets.macosX64, subproject.kotlin.presets.linuxX64, subproject.kotlin.presets.mingwX64)
): KotlinTargetPreset<*> {
if (whitelist.isEmpty())
throw Exception("Preset whitelist must not be empty in Kotlin/Native ${subproject.displayName}.")
val presetCandidate = when {
PlatformInfo.isMac() -> subproject.kotlin.presets.macosX64
PlatformInfo.isLinux() -> subproject.kotlin.presets.linuxX64
PlatformInfo.isWindows() -> subproject.kotlin.presets.mingwX64
else -> null
}
return if (presetCandidate != null && presetCandidate in whitelist)
presetCandidate
else
throw Exception("Host OS '$hostOs' is not supported in Kotlin/Native ${subproject.displayName}.")
}
fun getNativeProgramExtension(): String = when {
PlatformInfo.isMac() -> ".kexe"
PlatformInfo.isLinux() -> ".kexe"
PlatformInfo.isWindows() -> ".exe"
else -> error("Unknown host")
}
fun getFileSize(filePath: String): Long? {
val file = File(filePath)
return if (file.exists()) file.length() else null
}
fun getCodeSizeBenchmark(programName: String, filePath: String): BenchmarkResult {
val codeSize = getFileSize(filePath)
return BenchmarkResult(programName,
codeSize?. let { BenchmarkResult.Status.PASSED } ?: run { BenchmarkResult.Status.FAILED },
codeSize?.toDouble() ?: 0.0, BenchmarkResult.Metric.CODE_SIZE, codeSize?.toDouble() ?: 0.0, 1, 0)
}
fun toCodeSizeBenchmark(metricDescription: String, status: String, programName: String): BenchmarkResult {
if (!metricDescription.startsWith("CODE_SIZE")) {
error("Wrong metric is used as code size.")
}
val codeSize = metricDescription.split(' ')[1].toDouble()
return BenchmarkResult(programName,
if (status == "PASSED") BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED,
codeSize, BenchmarkResult.Metric.CODE_SIZE, codeSize, 1, 0)
}
// Create benchmarks json report based on information get from gradle project
fun createJsonReport(projectProperties: Map<String, Any>): String {
fun getValue(key: String): String = projectProperties[key] as? String ?: "unknown"
val machine = Environment.Machine(getValue("cpu"), getValue("os"))
val jdk = Environment.JDKInstance(getValue("jdkVersion"), getValue("jdkVendor"))
val env = Environment(machine, jdk)
val flags = (projectProperties["flags"] ?: emptyList<String>()) as List<String>
val backend = Compiler.Backend(Compiler.backendTypeFromString(getValue("type"))!! ,
getValue("compilerVersion"), flags)
val kotlin = Compiler(backend, getValue("kotlinVersion"))
val benchDesc = getValue("benchmarks")
val benchmarksArray = JsonTreeParser.parse(benchDesc)
val benchmarks = parseBenchmarksArray(benchmarksArray)
.union(projectProperties["compileTime"] as List<BenchmarkResult>).union(
listOf(projectProperties["codeSize"] as? BenchmarkResult).filterNotNull()).toList()
val report = BenchmarksReport(env, benchmarks, kotlin)
return report.toJson()
}
fun mergeReports(reports: List<File>): String {
val reportsToMerge = reports.filter { it.exists() }.map {
val json = it.inputStream().bufferedReader().use { it.readText() }
val reportElement = JsonTreeParser.parse(json)
BenchmarksReport.create(reportElement)
}
val structuredReports = mutableMapOf<String, MutableList<BenchmarksReport>>()
reportsToMerge.map { it.compiler.backend.flags.joinToString() to it }.forEach {
structuredReports.getOrPut(it.first) { mutableListOf<BenchmarksReport>() }.add(it.second)
}
val jsons = structuredReports.map { (_, value) -> value.reduce { result, it -> result + it }.toJson() }
return when(jsons.size) {
0 -> ""
1 -> jsons[0]
else -> jsons.joinToString(prefix = "[", postfix = "]")
}
}
fun getCompileOnlyBenchmarksOpts(project: Project, defaultCompilerOpts: List<String>): List<String> {
val dist = project.file(project.findProperty("kotlin.native.home") ?: "dist")
val useCache = !project.hasProperty("disableCompilerCaches")
val cacheOption = "-Xcache-directory=$dist/klib/cache/${HostManager.host.name}-gSTATIC"
.takeIf { useCache && PlatformInfo.isMac() } // TODO: remove target condition when we have cache support for other targets.
return (project.findProperty("nativeBuildType") as String?)?.let {
if (it.equals("RELEASE", true))
listOf("-opt")
else if (it.equals("DEBUG", true))
listOfNotNull("-g", cacheOption)
else listOf()
} ?: defaultCompilerOpts + listOfNotNull(cacheOption?.takeIf { !defaultCompilerOpts.contains("-opt") })
}
// Find file with set name in directory.
fun findFile(fileName: String, directory: String): String? =
File(directory).walkTopDown().filter { !it.absolutePath.contains(".dSYM") }
.find { it.name == fileName }?.getAbsolutePath()
fun uploadFileToArtifactory(url: String, project: String, artifactoryFilePath: String,
filePath: String, password: String) {
val uploadUrl = "$url/$project/$artifactoryFilePath"
sendUploadRequest(uploadUrl, filePath, extraHeaders = listOf(Pair("X-JFrog-Art-Api", password)))
}
fun sendUploadRequest(url: String, fileName: String, username: String? = null, password: String? = null,
extraHeaders: List<Pair<String, String>> = emptyList()) {
val uploadingFile = File(fileName)
val connection = URL(url).openConnection() as HttpURLConnection
connection.doOutput = true
connection.doInput = true
connection.requestMethod = "PUT"
connection.setRequestProperty("Content-type", "text/plain")
if (username != null && password != null) {
val auth = Base64.getEncoder().encode((username + ":" + password).toByteArray()).toString(Charsets.UTF_8)
connection.addRequestProperty("Authorization", "Basic $auth")
}
extraHeaders.forEach {
connection.addRequestProperty(it.first, it.second)
}
try {
connection.connect()
BufferedOutputStream(connection.outputStream).use { output ->
BufferedInputStream(FileInputStream(uploadingFile)).use { input ->
input.copyTo(output)
}
}
val response = connection.responseMessage
println("Upload request ended with ${connection.responseCode} - $response")
} catch (t: Throwable) {
error("Couldn't upload file $fileName to $url")
}
}
// A short-cut to add a Kotlin/Native run task.
@JvmOverloads
fun createRunTask(
subproject: Project,
name: String,
linkTask: Task,
executable: String,
outputFileName: String
): Task {
return subproject.tasks.create(name, RunKotlinNativeTask::class.java, linkTask, executable, outputFileName)
}
fun getJvmCompileTime(subproject: Project,programName: String): BenchmarkResult =
TaskTimerListener.getTimerListenerOfSubproject(subproject)
.getBenchmarkResult(programName, listOf("compileKotlinMetadata", "jvmJar"))
@JvmOverloads
fun getNativeCompileTime(subproject: Project, programName: String,
tasks: List<String> = listOf("linkBenchmarkReleaseExecutableNative")): BenchmarkResult =
TaskTimerListener.getTimerListenerOfSubproject(subproject).getBenchmarkResult(programName, tasks)
fun getCompileBenchmarkTime(subproject: Project,
programName: String, tasksNames: Iterable<String>,
repeats: Int, exitCodes: Map<String, Int>) =
(1..repeats).map { number ->
var time = 0.0
var status = BenchmarkResult.Status.PASSED
tasksNames.forEach {
time += TaskTimerListener.getTimerListenerOfSubproject(subproject).getTime("$it$number")
status = if (exitCodes["$it$number"] != 0) BenchmarkResult.Status.FAILED else status
}
BenchmarkResult(programName, status, time, BenchmarkResult.Metric.COMPILE_TIME, time, number, 0)
}.toList()
fun toCompileBenchmark(metricDescription: String, status: String, programName: String): BenchmarkResult {
if (!metricDescription.startsWith("COMPILE_TIME")) {
error("Wrong metric is used as compile time.")
}
val time = metricDescription.split(' ')[1].toDouble()
return BenchmarkResult(programName,
if (status == "PASSED") BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED,
time, BenchmarkResult.Metric.COMPILE_TIME, time, 1, 0)
}
// Class time tracker for all tasks.
class TaskTimerListener: TaskExecutionListener {
companion object {
internal val timerListeners = mutableMapOf<String, TaskTimerListener>()
internal fun getTimerListenerOfSubproject(subproject: Project) =
timerListeners[subproject.name] ?: error("TimeListener for project ${subproject.name} wasn't set")
}
val tasksTimes = mutableMapOf<String, Double>()
fun getBenchmarkResult(programName: String, tasksNames: List<String>): BenchmarkResult {
val time = tasksNames.map { tasksTimes[it] ?: 0.0 }.sum()
// TODO get this info from gradle plugin with exit code end stacktrace.
val status = tasksNames.map { tasksTimes.containsKey(it) }.reduce { a, b -> a && b }
return BenchmarkResult(programName,
if (status) BenchmarkResult.Status.PASSED else BenchmarkResult.Status.FAILED,
time, BenchmarkResult.Metric.COMPILE_TIME, time, 1, 0)
}
fun getTime(taskName: String) = tasksTimes[taskName] ?: 0.0
private var startTime = System.nanoTime()
override fun beforeExecute(task: Task) {
startTime = System.nanoTime()
}
override fun afterExecute(task: Task, taskState: TaskState) {
tasksTimes[task.name] = (System.nanoTime() - startTime) / 1000.0
}
}
fun addTimeListener(subproject: Project) {
val listener = TaskTimerListener()
TaskTimerListener.timerListeners.put(subproject.name, listener)
subproject.gradle.addListener(listener)
}
@@ -0,0 +1,87 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.TaskAction
import org.jetbrains.kotlin.klib.metadata.CInteropComparisonConfig
import org.jetbrains.kotlin.klib.metadata.MetadataCompareResult
import org.jetbrains.kotlin.klib.metadata.compareKlibMetadata
import org.jetbrains.kotlin.klib.metadata.expandFail
import org.jetbrains.kotlin.konan.target.HostManager
import java.io.File
/**
* Produces 2 interop libraries from the given [defFile].
* One in legacy `sourcecode` mode and another one in `metadata` mode.
* Then structurally compares metadata of the produced klibs.
*
*
* Motivation for this kind of tests:
*
* `sourcecode` mode generates Kotlin source file that is passes to a
* regular Kotlin/Native compiler.
* Thus, the produced metadata is correct (from compiler's point of view)
* because it is produced by the compiler itself.
*
* `metadata` mode generates metadata directly from parsed Clang AST.
* Since another algorithm is used, produced metadata may be incorrect.
*
* So we have to check that these two are more or less the same.
*/
open class MetadataComparisonTest : DefaultTask() {
private enum class Mode {
METADATA, SOURCECODE
}
/**
* Path to a cinterop *.def file.
*/
@Input
lateinit var defFile: String
@TaskAction
fun run() {
val metadataLibrary = cinterop(project.file(defFile), Mode.METADATA)
val sourcecodeLibrary = cinterop(project.file(defFile), Mode.SOURCECODE)
compareKlibMetadata(CInteropComparisonConfig(), sourcecodeLibrary.absolutePath, metadataLibrary.absolutePath).let { result ->
if (result is MetadataCompareResult.Fail) {
val message = StringBuilder().also {
expandFail(result, it::appendln)
}.toString()
throw TestFailedException(message)
}
}
}
private fun cinterop(defFile: File, mode: Mode): File {
val dist = project.kotlinNativeDist
val output = "${project.buildDir.absolutePath}/${defFile.nameWithoutExtension}_$mode"
val tool = if (HostManager.hostIsMingw) "cinterop.bat" else "cinterop"
val cinterop = File("${dist.canonicalPath}/bin/$tool").absolutePath
val args = listOf(
"-def", defFile.absolutePath,
"-mode", mode.name.toLowerCase(),
"-target", project.testTarget.visibleName,
"-no-default-libs", "-no-endorsed-libs",
"-o", output
)
runProcess(localExecutor(project), cinterop, args).let { result ->
if (result.exitCode != 0) {
println("""
cinterop failed.
exitCode: ${result.exitCode}
stdout:
${result.stdOut}
stderr:
${result.stdErr}
""".trimIndent())
}
}
return File(output)
}
}
@@ -0,0 +1,60 @@
package org.jetbrains.kotlin
import org.jetbrains.kotlin.konan.target.*
import org.jetbrains.kotlin.konan.util.*
import org.gradle.api.Project
object PlatformInfo {
@JvmStatic
fun isMac() = HostManager.hostIsMac
@JvmStatic
fun isWindows() = HostManager.hostIsMingw
@JvmStatic
fun isLinux() = HostManager.hostIsLinux
@JvmStatic
fun isAppleTarget(project: Project): Boolean {
val target = getTarget(project)
return target.family.isAppleFamily
}
@JvmStatic
fun isAppleTarget(target: KonanTarget): Boolean {
return target.family.isAppleFamily
}
@JvmStatic
fun isWindowsTarget(project: Project) = getTarget(project).family == Family.MINGW
@JvmStatic
fun isWasmTarget(project: Project) =
getTarget(project).family == Family.WASM
@JvmStatic
fun getTarget(project: Project): KonanTarget {
val platformManager = project.rootProject.platformManager
val targetName = project.project.testTarget.name
return platformManager.targetManager(targetName).target
}
@JvmStatic
fun checkXcodeVersion(project: Project) {
val properties = PropertiesProvider(project)
val requiredMajorVersion = properties.xcodeMajorVersion
if (!DependencyProcessor.isInternalSeverAvailable
&& properties.checkXcodeVersion
&& requiredMajorVersion != null
) {
val currentXcodeVersion = Xcode.current.version
val currentMajorVersion = currentXcodeVersion.splitToSequence('.').first()
if (currentMajorVersion != requiredMajorVersion) {
throw IllegalStateException(
"Incorrect Xcode version: ${currentXcodeVersion}. Required major Xcode version is ${requiredMajorVersion}."
)
}
}
}
fun unsupportedPlatformException() = TargetSupportException()
}
@@ -0,0 +1,31 @@
package org.jetbrains.kotlin
import org.gradle.api.Project
import java.util.*
class PropertiesProvider(val project: Project) {
private val localProperties by lazy {
Properties().apply {
project.file("local.properties").takeIf { it.isFile }?.inputStream()?.use {
load(it)
}
}
}
fun findProperty(name: String): Any? =
project.findProperty(name) ?: localProperties.getProperty(name)
fun getProperty(name: String): Any =
findProperty(name) ?: throw IllegalArgumentException("No such property: $name")
fun hasProperty(name: String): Boolean =
project.hasProperty(name) || localProperties.containsKey(name)
val xcodeMajorVersion: String?
get() = findProperty("xcodeMajorVersion") as String?
val checkXcodeVersion: Boolean
get() = findProperty("checkXcodeVersion")?.let {
it == "true"
} ?: true
}
@@ -0,0 +1,173 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import groovy.lang.Closure
import org.gradle.api.Action
import org.gradle.api.DefaultTask
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.TaskAction
import com.ullink.slack.simpleslackapi.impl.SlackSessionFactory
import org.jetbrains.report.json.*
import java.io.FileInputStream
import java.io.IOException
import java.io.File
import java.util.concurrent.TimeUnit
import java.net.HttpURLConnection
import java.net.URL
import java.util.Base64
import java.util.Properties
/**
* Task to produce regressions report and send it to slack. Requires a report with current benchmarks result
* and path to analyzer tool
*
* @property currentBenchmarksReportFile path to file with becnhmarks result
* @property analyzer path to analyzer tool
* @property htmlReport name of result html report
* @property defaultBranch name of default branch
* @property summaryFile name of file with short summary
* @property bundleBuild property to show if current build is full or not
*/
open class RegressionsReporter : DefaultTask() {
val slackUsers = mapOf(
"olonho" to "nikolay.igotti",
"nikolay.igotti" to "nikolay.igotti",
"ilya.matveev" to "ilya.matveev",
"ilmat192" to "ilya.matveev",
"vasily.v.levchenko" to "minamoto",
"vasily.levchenko" to "minamoto",
"alexander.gorshenev" to "alexander.gorshenev",
"igor.chevdar" to "igor.chevdar",
"pavel.punegov" to "Pavel Punegov",
"dmitriy.dolovov" to "dmitriy.dolovov",
"svyatoslav.scherbina" to "svyatoslav.scherbina",
"sbogolepov" to "sergey.bogolepov",
"Alexey.Zubakov" to "Alexey.Zubakov",
"kirill.shmakov" to "kirill.shmakov",
"elena.lepilkina" to "elena.lepilkina")
@Input
lateinit var currentBenchmarksReportFile: String
@Input
lateinit var analyzer: String
@Input
lateinit var htmlReport: String
@Input
lateinit var defaultBranch: String
@Input
lateinit var summaryFile: String
@Input
var bundleBuild: Boolean = false
private fun tabUrl(buildId: String, buildTypeId: String, tab: String) =
"$teamCityUrl/viewLog.html?buildId=$buildId&buildTypeId=$buildTypeId&tab=$tab"
private fun testReportUrl(buildId: String, buildTypeId: String) =
tabUrl(buildId, buildTypeId, "testsInfo")
private fun previousBuildLocator(buildTypeId: String, branchName: String) =
"buildType:id:$buildTypeId,branch:name:$branchName,status:SUCCESS,state:finished,count:1"
private fun changesListUrl(buildLocator: String) =
"$teamCityUrl/app/rest/changes/?locator=build:$buildLocator"
private fun getCommits(buildLocator: String, user: String, password: String): CommitsList {
val changes = try {
sendGetRequest(changesListUrl(buildLocator), user, password)
} catch (t: Throwable) {
error("Try to get commits! TeamCity is unreachable!")
}
return CommitsList(JsonTreeParser.parse(changes))
}
@TaskAction
fun run() {
// Get TeamCity properties.
val teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") ?:
error("Can't load teamcity config!")
val buildProperties = Properties()
buildProperties.load(FileInputStream(teamcityConfig))
val buildId = buildProperties.getProperty("teamcity.build.id")
val buildTypeId = buildProperties.getProperty("teamcity.buildType.id")
val buildNumber = buildProperties.getProperty("build.number")
val user = buildProperties.getProperty("teamcity.auth.userId")
val password = buildProperties.getProperty("teamcity.auth.password")
// Get branch.
val currentBuild = getBuild("id:$buildId", user, password)
val branch = getBuildProperty(currentBuild,"branchName")
val testReportUrl = testReportUrl(buildId, buildTypeId)
// Get previous build on branch.
val builds = getBuild(previousBuildLocator(buildTypeId,branch), user, password)
// Get changes description.
val changesList = getCommits("id:$buildId", user, password)
val changesInfo = "*Changes* in branch *$branch:*\n" + buildString {
changesList.commits.forEach {
append(" - Change ${it.revision} by ${it.developer} (details: ${it.webUrlWithDescription})\n")
}
}
// File name on Artifactory is the same as current.
val artifactoryFileName = currentBenchmarksReportFile.substringAfterLast("/")
// Get compare to build.
val compareToBuild = getBuild(previousBuildLocator(buildTypeId, defaultBranch), user, password)
val compareToBuildLink = getBuildProperty(compareToBuild,"webUrl")
val compareToBuildNumber = getBuildProperty(compareToBuild,"number")
val target = System.getProperty("os.name").replace("\\s".toRegex(), "")
// Generate comparison report.
val output = arrayOf("$analyzer", "-r", "html", "$currentBenchmarksReportFile", "artifactory:$compareToBuildNumber:$target:$artifactoryFileName", "-o", "$htmlReport")
.runCommand()
if (output.contains("Uncaught exception")) {
error("Error during comparasion of $currentBenchmarksReportFile and " +
"artifactory:$compareToBuildNumber:$target:$artifactoryFileName with $analyzer! " +
"Please check files existance and their correctness.")
}
arrayOf("$analyzer", "-r", "statistics", "$currentBenchmarksReportFile", "artifactory:$compareToBuildNumber:$target:$artifactoryFileName", "-o", "$summaryFile")
.runCommand()
val reportLink = "https://kotlin-native-perf-summary.labs.jb.gg/?target=$target&build=$buildNumber"
val detailedReportLink = "https://kotlin-native-performance.labs.jb.gg/?" +
"report=artifactory:$buildNumber:$target:$artifactoryFileName&" +
"compareTo=artifactory:$compareToBuildNumber:$target:$artifactoryFileName"
val title = "\n*Performance report for target $target (build $buildNumber)* - $reportLink\n" +
"*Detailed info - * $detailedReportLink"
val header = "$title\n$changesInfo\n\nCompare to build $compareToBuildNumber: $compareToBuildLink\n\n"
val footer = "*Benchmarks statistics:* $testReportUrl"
val message = "$header\n$footer\n"
// Send to channel or user directly.
val session = SlackSessionFactory.createWebSocketSlackSession(buildProperties.getProperty("konan-reporter-token"))
session.connect()
if (branch == defaultBranch) {
if (bundleBuild) {
val channel = session.findChannelByName(buildProperties.getProperty("konan-channel-name"))
session.sendMessage(channel, message)
}
}
session.disconnect()
}
}
@@ -0,0 +1,105 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import groovy.lang.Closure
import org.gradle.api.Action
import org.gradle.api.DefaultTask
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.TaskAction
import com.ullink.slack.simpleslackapi.impl.SlackSessionFactory
import com.ullink.slack.simpleslackapi.SlackAttachment
import com.ullink.slack.simpleslackapi.SlackPreparedMessage
import java.io.FileInputStream
import java.io.File
import java.util.Properties
/**
* Task to produce regressions report and send it to slack. Requires a report with current benchmarks result
* and path to analyzer tool
*
* @property targetsResultFiles map with pathes to results of each target
*/
open class RegressionsSummaryReporter : DefaultTask() {
@Input
lateinit var targetsResultFiles: Map<String, String>
val performanceServer = "https://kotlin-native-perf-summary.labs.jb.gg/"
@TaskAction
fun run() {
// Get TeamCity properties.
val teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE") ?:
error("Can't load teamcity config!")
val buildProperties = Properties()
buildProperties.load(FileInputStream(teamcityConfig))
val buildId = buildProperties.getProperty("teamcity.build.id")
val buildTypeId = buildProperties.getProperty("teamcity.buildType.id")
val buildNumber = buildProperties.getProperty("build.number")
val results = mutableMapOf<String, MutableMap<String, String>>()
// Parse and merge results from all targets.
targetsResultFiles.forEach { (key, value) ->
val file = File(value)
if (file.exists()) {
file.forEachLine {
val matchResult = "(\\w+)\\s*:\\s*(\\w+)".toRegex().find(it)
val propertyName = matchResult?.groups?.get(1)?.value
val propertyValue = matchResult?.groups?.get(2)?.value
if (propertyName != null && propertyValue != null) {
results[propertyName]?.let { it[key] = propertyValue }
?: run { results.put(propertyName, mutableMapOf(key to propertyValue)) }
}
}
}
}
val message = buildString {
append("*Performance summary on charts* - $performanceServer\n")
results.forEach { (property, targets) ->
append("$property: ${targets.map {(target, value) -> "$value ($target)"}.joinToString(" | ")}\n")
}
}
val summaryStatus = results["status"]?.values?.fold("STABLE") {summary, element ->
when {
summary == "FAILED" || element == "FAILED" -> "FAILED"
summary == "FIXED" || element == "FIXED" -> "FIXED"
summary == "STABLE" -> element
summary == element -> summary
else -> "UNSTABLE"
}
}
val attachement = SlackAttachment()
with (attachement) {
setTitle("Performance Summary (build $buildNumber)")
setTitleLink("https://buildserver.labs.intellij.net/viewLog.html?buildId=$buildId&buildTypeId=$buildTypeId")
setText(message)
if (summaryStatus == "FIXED" || summaryStatus == "IMPROVED") {
setColor("#36a64f")
} else if (summaryStatus == "FAILED" || summaryStatus == "REGRESSED") {
setColor("#ff0000")
}
}
// Send to channel or user directly.
val session = SlackSessionFactory.createWebSocketSlackSession(buildProperties.getProperty("konan-reporter-token"))
session.connect()
val channel = session.findChannelByName(buildProperties.getProperty("konan-channel-name"))
val preparedMessage = SlackPreparedMessage.Builder()
.addAttachment(attachement)
.build()
session.sendMessage(channel, preparedMessage)
session.disconnect()
}
}
@@ -0,0 +1,109 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin
import com.ullink.slack.simpleslackapi.impl.SlackSessionFactory
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.TaskAction
import java.io.FileInputStream
import java.util.*
internal object Tc {
private val teamcityConfig = System.getenv("TEAMCITY_BUILD_PROPERTIES_FILE")
val enabled:Boolean = (teamcityConfig != null)
private val buildConfig by lazy {
teamcityConfig ?: return@lazy null
val properties = Properties()
properties.load(FileInputStream(teamcityConfig))
properties
}
val buildId = buildConfig?.getProperty("teamcity.build.id")
val buildTypeId = buildConfig?.getProperty("teamcity.buildType.id")
val konanReporterToken = buildConfig?.getProperty("konan-reporter-token")
val konanChannelName = buildConfig?.getProperty("konan-channel-name")
}
private fun buildLogUrlTab(buildId: String?, buildTypeId: String?): String = tabUrl(buildId, buildTypeId, "buildLog")
private fun tabUrl(buildId: String?, buildTypeId: String?, tab: String?): String =
"http://buildserver.labs.intellij.net/viewLog.html?buildId=$buildId&buildTypeId=$buildTypeId&tab=$tab"
private fun testReportUrl(buildId: String?, buildTypeId: String?): String = tabUrl(buildId, buildTypeId, "testsInfo")
private fun sendTextToSlack(report: String) {
with(SlackSessionFactory.createWebSocketSlackSession(Tc.konanReporterToken)) {
connect()
sendMessage(findChannelByName(Tc.konanChannelName),
"Hello, аборигены Котлина!\n текущий статус:\n$report")
disconnect()
}
}
private fun reportEpilogue(): String {
val logUrl = buildLogUrlTab(Tc.buildId, Tc.buildTypeId)
val testReportUrl = testReportUrl(Tc.buildId, Tc.buildTypeId)
return "\nlog url: $logUrl\ntest report url: $testReportUrl"
}
private val Statistics.report
get() = "total: $total\npassed: $passed\nfailed: $failed\nerror: $error\nskipped: $skipped"
private val Statistics.oneLineReport
get() = "(total: $total, passed: $passed, failed: $failed, error: $error, skipped: $skipped)"
open class Reporter : DefaultTask() {
@Input
lateinit var reportHome: String
@TaskAction
fun report() {
val reportJson = loadReport("$reportHome/external/results.json")
val report: String =
"${reportJson.statistics.report}\n ${reportEpilogue()}"
project.logger.info(report)
if (doSlackSending())
sendTextToSlack(report)
}
}
private fun DefaultTask.doSlackSending() = !project.hasProperty("build.reporter.noSlack")
|| !project.property("build.reporter.noSlack").toString().toBoolean()
open class NightlyReporter: DefaultTask() {
@Input
lateinit var externalMacosReport:String
@Input
lateinit var externalLinuxReport:String
@Input
lateinit var externalWindowsReport:String
@TaskAction
fun report() {
val externalMacosJsonReport = loadReport("${project.rootDir.absolutePath}/$externalMacosReport")
val externalLinuxJsonReport = loadReport("${project.rootDir.absolutePath}/$externalLinuxReport")
val externalWindowsJsonReport = loadReport("${project.rootDir.absolutePath}/$externalWindowsReport")
val report = buildString {
append("Mac OS ")
appendln(externalMacosJsonReport.statistics.oneLineReport)
append("Linux ")
appendln(externalLinuxJsonReport.statistics.oneLineReport)
append("Windows ")
appendln(externalWindowsJsonReport.statistics.oneLineReport)
appendln(reportEpilogue())
}
project.logger.info(report)
if (doSlackSending())
sendTextToSlack(report)
}
}
@@ -0,0 +1,125 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import org.gradle.api.tasks.JavaExec
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.options.Option
import org.gradle.api.tasks.Input
import org.jetbrains.kotlin.benchmark.LogLevel
import org.jetbrains.kotlin.benchmark.Logger
import org.jetbrains.report.json.*
import java.io.ByteArrayOutputStream
import java.io.File
data class ExecParameters(val warmupCount: Int, val repeatCount: Int,
val filterArgs: List<String>, val filterRegexArgs: List<String>,
val verbose: Boolean, val outputFileName: String?)
open class RunJvmTask: JavaExec() {
@Input
@Option(option = "filter", description = "Benchmarks to run (comma-separated)")
var filter: String = ""
@Input
@Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)")
var filterRegex: String = ""
@Input
@Option(option = "verbose", description = "Verbose mode of running benchmarks")
var verbose: Boolean = false
@Input
var warmupCount: Int = 0
@Input
var repeatCount: Int = 0
@Input
var repeatingType = BenchmarkRepeatingType.INTERNAL
@Input
var outputFileName: String? = null
private var predefinedArgs: List<String> = emptyList()
private fun executeTask(execParameters: ExecParameters): String {
// Firstly clean arguments.
setArgs(emptyList())
args(predefinedArgs)
args(execParameters.filterArgs)
args(execParameters.filterRegexArgs)
args("-w", execParameters.warmupCount)
args("-r", execParameters.repeatCount)
if (execParameters.verbose) {
args("-v")
}
execParameters.outputFileName?.let { args("-o", outputFileName) }
standardOutput = ByteArrayOutputStream()
super.exec()
return standardOutput.toString()
}
private fun getBenchmarksList(filterArgs: List<String>, filterRegexArgs: List<String>): List<String> {
// Firstly clean arguments.
setArgs(emptyList())
args("list")
standardOutput = ByteArrayOutputStream()
super.exec()
val benchmarks = standardOutput.toString().lines()
val regexes = filterRegexArgs.map { it.toRegex() }
return if (filterArgs.isNotEmpty() || regexes.isNotEmpty()) {
benchmarks.filter { benchmark -> benchmark in filterArgs || regexes.any { it.matches(benchmark) } }
} else benchmarks.filter { !it.isEmpty() }
}
private fun execSeparateBenchmarkRepeatedly(benchmark: String): List<String> {
// Logging with application should be done only in case it controls running benchmarks itself.
// Although it's a responsibility of gradle task.
val logger = if (verbose) Logger(LogLevel.DEBUG) else Logger()
logger.log("Warm up iterations for benchmark $benchmark\n")
for (i in 0.until(warmupCount)) {
executeTask(ExecParameters(0, 1, listOf("-f", benchmark),
emptyList(), false, null))
}
val result = mutableListOf<String>()
logger.log("Running benchmark $benchmark ")
for (i in 0.until(repeatCount)) {
logger.log(".", usePrefix = false)
val benchmarkReport = JsonTreeParser.parse(
executeTask(ExecParameters(0, 1, listOf("-f", benchmark),
emptyList(), false, null)
).removePrefix("[").removeSuffix("]")
).jsonObject
val modifiedBenchmarkReport = JsonObject(HashMap(benchmarkReport.content).apply {
put("repeat", JsonLiteral(i))
put("warmup", JsonLiteral(warmupCount))
})
result.add(modifiedBenchmarkReport.toString())
}
logger.log("\n", usePrefix = false)
return result
}
private fun execBenchmarksRepeatedly(filterArgs: List<String>, filterRegexArgs: List<String>) {
val benchmarksToRun = getBenchmarksList(filterArgs, filterRegexArgs)
val results = benchmarksToRun.flatMap { benchmark ->
execSeparateBenchmarkRepeatedly(benchmark)
}
File(outputFileName).printWriter().use { out ->
out.println("[${results.joinToString(",")}]")
}
}
@TaskAction
override fun exec() {
assert(outputFileName != null) { "Output file name should be always set" }
predefinedArgs = args ?: emptyList()
val filterArgs = filter.splitCommaSeparatedOption("-f")
val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr")
when (repeatingType) {
BenchmarkRepeatingType.INTERNAL -> executeTask(
ExecParameters(warmupCount, repeatCount, filterArgs, filterRegexArgs, verbose, outputFileName)
)
BenchmarkRepeatingType.EXTERNAL -> execBenchmarksRepeatedly(filterArgs, filterRegexArgs)
}
}
}
@@ -0,0 +1,131 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
import org.gradle.api.DefaultTask
import org.gradle.api.Task
import org.jetbrains.kotlin.benchmark.Logger
import org.jetbrains.kotlin.benchmark.LogLevel
import org.jetbrains.report.json.*
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.options.Option
import org.gradle.api.tasks.Input
import java.io.ByteArrayOutputStream
import java.io.File
import javax.inject.Inject
import kotlin.collections.HashMap
open class RunKotlinNativeTask @Inject constructor(private val linkTask: Task,
private val executable: String,
private val outputFileName: String
) : DefaultTask() {
@Input
@Option(option = "filter", description = "Benchmarks to run (comma-separated)")
var filter: String = ""
@Input
@Option(option = "filterRegex", description = "Benchmarks to run, described by regular expressions (comma-separated)")
var filterRegex: String = ""
@Input
@Option(option = "verbose", description = "Verbose mode of running benchmarks")
var verbose: Boolean = false
@Input
var warmupCount: Int = 0
@Input
var repeatCount: Int = 0
@Input
var repeatingType = BenchmarkRepeatingType.INTERNAL
private val argumentsList = mutableListOf<String>()
init {
this.dependsOn += linkTask.name
this.finalizedBy("konanJsonReport")
}
fun depends(taskName: String) {
this.dependsOn += taskName
}
fun args(vararg arguments: String) {
argumentsList.addAll(arguments.toList())
}
private fun execBenchmarkOnce(benchmark: String, warmupCount: Int, repeatCount: Int) : String {
val output = ByteArrayOutputStream()
val useCset = project.findProperty("useCset")?.toString()?.toBoolean() ?: false
project.exec {
if (useCset) {
it.executable = "cset"
it.args("shield", "--exec", "--", executable)
} else {
it.executable = executable
}
it.args(argumentsList)
it.args("-f", benchmark)
// Logging with application should be done only in case it controls running benchmarks itself.
// Although it's a responsibility of gradle task.
if (verbose && repeatingType == BenchmarkRepeatingType.INTERNAL) {
it.args("-v")
}
it.args("-w", warmupCount.toString())
it.args("-r", repeatCount.toString())
it.standardOutput = output
}
return output.toString().substringAfter("[").removeSuffix("]")
}
private fun execBenchmarkRepeatedly(benchmark: String, warmupCount: Int, repeatCount: Int) : List<String> {
val logger = if (verbose) Logger(LogLevel.DEBUG) else Logger()
logger.log("Warm up iterations for benchmark $benchmark\n")
for (i in 0.until(warmupCount)) {
execBenchmarkOnce(benchmark, 0, 1)
}
val result = mutableListOf<String>()
logger.log("Running benchmark $benchmark ")
for (i in 0.until(repeatCount)) {
logger.log(".", usePrefix = false)
val benchmarkReport = JsonTreeParser.parse(execBenchmarkOnce(benchmark, 0, 1)).jsonObject
val modifiedBenchmarkReport = JsonObject(HashMap(benchmarkReport.content).apply {
put("repeat", JsonLiteral(i))
put("warmup", JsonLiteral(warmupCount))
})
result.add(modifiedBenchmarkReport.toString())
}
logger.log("\n", usePrefix = false)
return result
}
@TaskAction
fun run() {
val output = ByteArrayOutputStream()
project.exec {
it.executable = executable
it.args("list")
it.standardOutput = output
}
val benchmarks = output.toString().lines()
val filterArgs = filter.splitCommaSeparatedOption("-f")
val filterRegexArgs = filterRegex.splitCommaSeparatedOption("-fr")
val regexes = filterRegexArgs.map { it.toRegex() }
val benchmarksToRun = if (filterArgs.isNotEmpty() || regexes.isNotEmpty()) {
benchmarks.filter { benchmark -> benchmark in filterArgs || regexes.any { it.matches(benchmark) } }
} else benchmarks.filter { !it.isEmpty() }
val results = benchmarksToRun.flatMap { benchmark ->
when (repeatingType) {
BenchmarkRepeatingType.INTERNAL -> listOf(execBenchmarkOnce(benchmark, warmupCount, repeatCount))
BenchmarkRepeatingType.EXTERNAL -> execBenchmarkRepeatedly(benchmark, warmupCount, repeatCount)
}
}
File(outputFileName).printWriter().use { out ->
out.println("[${results.joinToString(",")}]")
}
}
}
@@ -0,0 +1,116 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin
import java.nio.file.Path
import java.nio.file.Paths
import java.util.regex.Pattern
private const val MODULE_DELIMITER = ",\\s*"
// This pattern is a copy from the kotlin/compiler/tests-common/tests/org/jetbrains/kotlin/test/TestFiles.java
private val FILE_OR_MODULE_PATTERN: Pattern = Pattern.compile("(?://\\s*MODULE:\\s*([^()\\n]+)(?:\\(([^()]+(?:" +
"$MODULE_DELIMITER[^()]+)*)\\))?\\s*(?:\\(([^()]+(?:$MODULE_DELIMITER[^()]+)*)\\))?\\s*)?//\\s*FILE:\\s*(.*)$",
Pattern.MULTILINE)
/**
* Creates test files from the given source file that may contain different test directives.
*
* @return list of test files [TestFile] to be compiled
*/
fun buildCompileList(source: Path, outputDirectory: String): List<TestFile> {
val result = mutableListOf<TestFile>()
val srcFile = source.toFile()
// Remove diagnostic parameters in external tests.
val srcText = srcFile.readText().replace(Regex("<!.*?!>(.*?)<!>")) { match -> match.groupValues[1] }
if (srcText.contains("// WITH_COROUTINES")) {
result.add(TestFile("helpers.kt", "$outputDirectory/helpers.kt",
createTextForHelpers(true), TestModule.support))
}
val matcher = FILE_OR_MODULE_PATTERN.matcher(srcText)
if (!matcher.find()) {
// There is only one file in the input
result.add(TestFile(srcFile.name, "$outputDirectory/${srcFile.name}", srcText))
} else {
// There are several files
var processedChars = 0
var module: TestModule = TestModule.default
var nextFileExists = true
while (nextFileExists) {
var moduleName = matcher.group(1)
val moduleDependencies = matcher.group(2)
val moduleFriends = matcher.group(3)
if (moduleName != null) {
moduleName = moduleName.trim { it <= ' ' }
module = TestModule("${srcFile.name}.$moduleName",
moduleDependencies.parseModuleList().map {
if (it != "support") "${srcFile.name}.$it" else it
},
moduleFriends.parseModuleList().map { "${srcFile.name}.$it" })
}
val fileName = matcher.group(4)
val filePath = "$outputDirectory/$fileName"
val start = processedChars
nextFileExists = matcher.find()
val end = if (nextFileExists) matcher.start() else srcText.length
val fileText = srcText.substring(start, end)
processedChars = end
if (fileName.endsWith(".kt")) {
result.add(TestFile(fileName, filePath, fileText, module))
}
}
}
return result
}
private fun String?.parseModuleList() = this
?.split(Pattern.compile(MODULE_DELIMITER), 0)
?: emptyList()
/**
* Test module from the test source declared by the [FILE_OR_MODULE_PATTERN].
* Module should have a [name] and could have [dependencies] on other modules and [friends].
*
* There are 2 predefined modules:
* - [default] that contains all sources that don't declare a module,
* - [support] for a helper sources like Coroutines support.
*/
data class TestModule(
val name: String,
val dependencies: List<String>,
val friends: List<String>
) {
fun isDefaultModule() = this == default || name.endsWith(".main")
companion object {
val default = TestModule("default", emptyList(), emptyList())
val support = TestModule("support", emptyList(), emptyList())
}
}
/**
* Represent a single test file that belongs to the [module].
*/
data class TestFile(val name: String,
val path: String,
var text: String = "",
val module: TestModule = TestModule.default
) {
/**
* Writes [text] to the file created from the [path].
*/
fun writeTextToFile() {
Paths.get(path).takeUnless { text.isEmpty() }?.run {
parent.toFile()
.takeUnless { it.exists() }
?.mkdirs()
toFile().writeText(text)
}
}
}
@@ -0,0 +1,351 @@
/*
* Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin
import org.gradle.api.Project
import org.gradle.api.Task
import org.jetbrains.kotlin.konan.properties.loadProperties
import org.jetbrains.kotlin.konan.properties.propertyList
import org.jetbrains.kotlin.konan.properties.saveProperties
import org.jetbrains.kotlin.konan.target.*
import org.jetbrains.kotlin.library.KLIB_PROPERTY_NATIVE_TARGETS
import java.io.File
import java.util.concurrent.TimeUnit
import java.net.HttpURLConnection
import java.net.URL
import java.util.Base64
import org.jetbrains.report.json.*
import java.nio.file.Path
import org.jetbrains.kotlin.konan.file.File as KFile
//region Project properties.
val Project.platformManager
get() = findProperty("platformManager") as PlatformManager
val Project.testTarget
get() = findProperty("target") as KonanTarget
val Project.verboseTest
get() = hasProperty("test_verbose")
val Project.testOutputRoot
get() = findProperty("testOutputRoot") as String
val Project.testOutputLocal
get() = (findProperty("testOutputLocal") as File).toString()
val Project.testOutputStdlib
get() = (findProperty("testOutputStdlib") as File).toString()
val Project.testOutputFramework
get() = (findProperty("testOutputFramework") as File).toString()
val Project.testOutputExternal
get() = (findProperty("testOutputExternal") as File).toString()
val Project.kotlinNativeDist
get() = this.rootProject.file(this.findProperty("org.jetbrains.kotlin.native.home")
?: this.findProperty("konan.home") ?: "dist")
@Suppress("UNCHECKED_CAST")
val Project.globalTestArgs: List<String>
get() = with(findProperty("globalTestArgs")) {
if (this is Array<*>) this.toList() as List<String>
else this as List<String>
}
val Project.testTargetSupportsCodeCoverage: Boolean
get() = this.testTarget.supportsCodeCoverage()
//endregion
/**
* Ad-hoc signing of the specified path.
*/
fun codesign(project: Project, path: String) {
check(HostManager.hostIsMac) { "Apple specific code signing" }
val (stdOut, stdErr, exitCode) = runProcess(executor = localExecutor(project), executable = "/usr/bin/codesign",
args = listOf("--verbose", "-s", "-", path))
check(exitCode == 0) { """
|Codesign failed with exitCode: $exitCode
|stdout: $stdOut
|stderr: $stdErr
""".trimMargin()
}
}
/**
* Creates a list of file paths to be compiled from the given [compile] list with regard to [exclude] list.
*/
fun Project.getFilesToCompile(compile: List<String>, exclude: List<String>): List<String> {
// convert exclude list to paths
val excludeFiles = exclude.map { project.file(it).absolutePath }.toList()
// create list of tests to compile
return compile.flatMap { f ->
project.file(f)
.walk()
.filter { it.isFile && it.name.endsWith(".kt") && !excludeFiles.contains(it.absolutePath) }
.map{ it.absolutePath }
.asIterable()
}
}
//region Task dependency.
fun Project.findKonanBuildTask(artifact: String, target: KonanTarget): Task =
tasks.getByName("compileKonan${artifact.capitalize()}${target.name.capitalize()}")
fun Project.dependsOnDist(taskName: String) {
project.tasks.getByName(taskName).dependsOnDist()
}
fun Task.dependsOnDist() {
val rootTasks = project.rootProject.tasks
// We don't build the compiler if a custom dist path is specified.
if (!(project.findProperty("useCustomDist") as Boolean)) {
dependsOn(rootTasks.getByName("dist"))
val target = project.testTarget
if (target != HostManager.host) {
// if a test_target property is set then tests should depend on a crossDist
// otherwise, runtime components would not be build for a target.
dependsOn(rootTasks.getByName("${target.name}CrossDist"))
}
}
}
/**
* Sets the same dependencies for the receiver task from the given [task]
*/
fun String.sameDependenciesAs(task: Task) {
val t = task.project.tasks.getByName(this)
t.sameDependenciesAs(task)
}
/**
* Sets the same dependencies for the receiver task from the given [task]
*/
fun Task.sameDependenciesAs(task: Task) {
val dependencies = task.dependsOn.toList() // save to the list, otherwise it will cause cyclic dependency.
this.dependsOn(dependencies)
}
/**
* Set dependency on [artifact] built by the Konan Plugin for the receiver task,
* also make [artifact] depend on `dist` and all dependencies of the task to make [artifact] execute before the task.
*/
fun Task.dependsOnKonanBuildingTask(artifact: String, target: KonanTarget) {
val buildTask = project.findKonanBuildTask(artifact, target)
buildTask.dependsOnDist()
buildTask.sameDependenciesAs(this)
dependsOn(buildTask)
}
//endregion
// Run command line from string.
fun Array<String>.runCommand(workingDir: File = File("."),
timeoutAmount: Long = 60,
timeoutUnit: TimeUnit = TimeUnit.SECONDS): String {
return try {
ProcessBuilder(*this)
.directory(workingDir)
.redirectOutput(ProcessBuilder.Redirect.PIPE)
.redirectError(ProcessBuilder.Redirect.PIPE)
.start().apply {
waitFor(timeoutAmount, timeoutUnit)
}.inputStream.bufferedReader().readText()
} catch (e: Exception) {
println("Couldn't run command ${this.joinToString(" ")}")
println(e.stackTrace.joinToString("\n"))
error(e.message!!)
}
}
fun String.splitCommaSeparatedOption(optionName: String) =
split("\\s*,\\s*".toRegex()).map {
if (it.isNotEmpty()) listOf(optionName, it) else listOf(null)
}.flatten().filterNotNull()
data class Commit(val revision: String, val developer: String, val webUrlWithDescription: String)
val teamCityUrl = "http://buildserver.labs.intellij.net"
// List of commits.
class CommitsList(data: JsonElement): ConvertedFromJson {
val commits: List<Commit>
init {
if (data !is JsonObject) {
error("Commits description is expected to be a json object!")
}
val changesElement = data.getOptionalField("change")
commits = changesElement?.let {
if (changesElement !is JsonArray) {
error("Change field is expected to be an array. Please, check source.")
}
changesElement.jsonArray.map {
with(it as JsonObject) {
Commit(elementToString(getRequiredField("version"), "version"),
elementToString(getRequiredField("username"), "username"),
elementToString(getRequiredField("webUrl"), "webUrl")
)
}
}
} ?: listOf<Commit>()
}
}
fun buildsUrl(buildLocator: String) =
"$teamCityUrl/app/rest/builds/?locator=$buildLocator"
fun getBuild(buildLocator: String, user: String, password: String) =
try {
sendGetRequest(buildsUrl(buildLocator), user, password)
} catch (t: Throwable) {
error("Try to get build! TeamCity is unreachable!")
}
fun sendGetRequest(url: String, username: String? = null, password: String? = null) : String {
val connection = URL(url).openConnection() as HttpURLConnection
if (username != null && password != null) {
val auth = Base64.getEncoder().encode(("$username:$password").toByteArray()).toString(Charsets.UTF_8)
connection.addRequestProperty("Authorization", "Basic $auth")
}
connection.setRequestProperty("Accept", "application/json");
connection.connect()
return connection.inputStream.use { it.reader().use { reader -> reader.readText() } }
}
fun getBuildProperty(buildJsonDescription: String, property: String) =
with(JsonTreeParser.parse(buildJsonDescription) as JsonObject) {
if (getPrimitive("count").int == 0) {
error("No build information on TeamCity for $buildJsonDescription!")
}
(getArray("build").getObject(0).getPrimitive(property) as JsonLiteral).unquoted()
}
@JvmOverloads
fun compileSwift(project: Project, target: KonanTarget, sources: List<String>, options: List<String>,
output: Path, fullBitcode: Boolean = false) {
val platform = project.platformManager.platform(target)
assert(platform.configurables is AppleConfigurables)
val configs = platform.configurables as AppleConfigurables
val compiler = configs.absoluteTargetToolchain + "/usr/bin/swiftc"
val swiftTarget = when (target) {
KonanTarget.IOS_X64 -> "x86_64-apple-ios" + configs.osVersionMin
KonanTarget.IOS_ARM32 -> "armv7-apple-ios" + configs.osVersionMin
KonanTarget.IOS_ARM64 -> "arm64-apple-ios" + configs.osVersionMin
KonanTarget.TVOS_X64 -> "x86_64-apple-tvos" + configs.osVersionMin
KonanTarget.TVOS_ARM64 -> "arm64-apple-tvos" + configs.osVersionMin
KonanTarget.MACOS_X64 -> "x86_64-apple-macosx" + configs.osVersionMin
KonanTarget.WATCHOS_X86 -> "i386-apple-watchos" + configs.osVersionMin
KonanTarget.WATCHOS_X64 -> "x86_64-apple-watchos" + configs.osVersionMin
else -> throw IllegalStateException("Test target $target is not supported")
}
val args = listOf("-sdk", configs.absoluteTargetSysRoot, "-target", swiftTarget) +
options + "-o" + output.toString() + sources +
if (fullBitcode) listOf("-embed-bitcode", "-Xlinker", "-bitcode_verify") else listOf("-embed-bitcode-marker")
val (stdOut, stdErr, exitCode) = runProcess(executor = localExecutor(project), executable = compiler, args = args)
println("""
|$compiler finished with exit code: $exitCode
|options: ${args.joinToString(separator = " ")}
|stdout: $stdOut
|stderr: $stdErr
""".trimMargin())
check(exitCode == 0) { "Compilation failed" }
check(output.toFile().exists()) { "Compiler swiftc hasn't produced an output file: $output" }
}
fun targetSupportsMimallocAllocator(targetName: String) =
HostManager().targetByName(targetName).supportsMimallocAllocator()
fun Project.mergeManifestsByTargets(source: File, destination: File) {
logger.info("Merging manifests: $source -> $destination")
val sourceFile = KFile(source.absolutePath)
val sourceProperties = sourceFile.loadProperties()
val destinationFile = KFile(destination.absolutePath)
val destinationProperties = destinationFile.loadProperties()
// check that all properties except for KLIB_PROPERTY_NATIVE_TARGETS are equivalent
val mismatchedProperties = (sourceProperties.keys + destinationProperties.keys)
.asSequence()
.map { it.toString() }
.filter { it != KLIB_PROPERTY_NATIVE_TARGETS }
.sorted()
.mapNotNull { propertyKey: String ->
val sourceProperty: String? = sourceProperties.getProperty(propertyKey)
val destinationProperty: String? = destinationProperties.getProperty(propertyKey)
when {
sourceProperty == null -> "\"$propertyKey\" is absent in $sourceFile"
destinationProperty == null -> "\"$propertyKey\" is absent in $destinationFile"
sourceProperty == destinationProperty -> {
// properties match, OK
null
}
sourceProperties.propertyList(propertyKey, escapeInQuotes = true).toSet() ==
destinationProperties.propertyList(propertyKey, escapeInQuotes = true).toSet() -> {
// properties match, OK
null
}
else -> "\"$propertyKey\" differ: [$sourceProperty] vs [$destinationProperty]"
}
}
.toList()
check(mismatchedProperties.isEmpty()) {
buildString {
appendln("Found mismatched properties while merging manifest files: $source -> $destination")
mismatchedProperties.joinTo(this, "\n")
}
}
// merge KLIB_PROPERTY_NATIVE_TARGETS property
val sourceNativeTargets = sourceProperties.propertyList(KLIB_PROPERTY_NATIVE_TARGETS)
val destinationNativeTargets = destinationProperties.propertyList(KLIB_PROPERTY_NATIVE_TARGETS)
val mergedNativeTargets = HashSet<String>().apply {
addAll(sourceNativeTargets)
addAll(destinationNativeTargets)
}
destinationProperties[KLIB_PROPERTY_NATIVE_TARGETS] = mergedNativeTargets.joinToString(" ")
destinationFile.saveProperties(destinationProperties)
}
fun Project.buildStaticLibrary(cSources: Collection<File>, output: File, objDir: File) {
delete(objDir)
delete(output)
val platform = platformManager.platform(testTarget)
objDir.mkdirs()
exec {
it.commandLine(platform.clang.clangC(
"-c",
*cSources.map { it.absolutePath }.toTypedArray()
))
it.workingDir(objDir)
}
output.parentFile.mkdirs()
exec {
it.commandLine(
"${platform.configurables.absoluteLlvmHome}/bin/llvm-ar",
"-rc",
output,
*fileTree(objDir).files.toTypedArray()
)
}
}
@@ -0,0 +1,53 @@
package org.jetbrains.kotlin
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.tasks.Input
import org.gradle.api.tasks.Optional
import org.gradle.api.tasks.wrapper.Wrapper
open class CustomWrapper : Wrapper() {
internal lateinit var mainWrapperTask: Wrapper
val mainWrapperVersion: String
@Input get() = mainWrapperTask.gradleVersion
val mainWrapperDistrSha256: String?
@Optional @Input get() = mainWrapperTask.distributionSha256Sum
}
open class WrappersExtension {
var projects = mutableListOf<Any>()
var distributionType = Wrapper.DistributionType.BIN
}
class GradleWrappers : Plugin<Project> {
override fun apply(project: Project): Unit = with(project) {
val mainWrapperTask = tasks.findByName("wrapper") as? Wrapper ?: return@with
val wrappers = extensions.create(
WrappersExtension::class.java,
"wrappers",
WrappersExtension::class.java
)
afterEvaluate {
wrappers.projects.map { file(it) }.forEach {
tasks.create("${it.name}Wrapper", CustomWrapper::class.java).apply {
this.mainWrapperTask = mainWrapperTask
jarFile = it.resolve("gradle/wrapper/gradle-wrapper.jar")
scriptFile = it.resolve("gradlew")
distributionType = wrappers.distributionType
mainWrapperTask.dependsOn(this)
// Get these parameters from the main wrapper task to support
// command line options like --gradle-version.
// Gradle doesn't provide access to the values passed in command line
// at the configuration phase, so we have to get these values at the execution phase.
doFirst {
gradleVersion = mainWrapperVersion
distributionSha256Sum = mainWrapperDistrSha256
}
}
}
}
}
}
@@ -0,0 +1,67 @@
package org.jetbrains.kotlin
import com.google.gson.annotations.Expose
import org.jetbrains.kotlin.konan.target.KonanTarget
import org.jetbrains.kotlin.konan.target.Xcode
import kotlin.math.min
/**
* Compares two strings assuming that both are representing numeric version strings.
* Examples of numeric version strings: "12.4.1.2", "9", "0.5".
*/
private fun compareStringsAsVersions(version1: String, version2: String): Int {
val version1 = version1.split('.').map { it.toInt() }
val version2 = version2.split('.').map { it.toInt() }
val minimalLength = min(version1.size, version2.size)
for (index in 0 until minimalLength) {
if (version1[index] < version2[index]) return -1
if (version1[index] > version2[index]) return 1
}
return version1.size.compareTo(version2.size)
}
/**
* Returns parsed output of `xcrun simctl list runtimes -j`.
*/
private fun Xcode.getSimulatorRuntimeDescriptors(): List<SimulatorRuntimeDescriptor> = gson.fromJson(simulatorRuntimes, ListRuntimesReport::class.java).runtimes
/**
* Returns first available simulator runtime for [target] with at least [osMinVersion] OS version.
* */
fun Xcode.getLatestSimulatorRuntimeFor(target: KonanTarget, osMinVersion: String): SimulatorRuntimeDescriptor? {
val osName = when (target) {
KonanTarget.IOS_X64 -> "iOS"
KonanTarget.WATCHOS_X64, KonanTarget.WATCHOS_X86 -> "watchOS"
KonanTarget.TVOS_X64 -> "tvOS"
else -> error("Unexpected simulator target: $target")
}
return getSimulatorRuntimeDescriptors().firstOrNull {
it.checkAvailability() && it.name.startsWith(osName) && compareStringsAsVersions(it.version, osMinVersion) >= 0
}
}
// Result of `xcrun simctl list runtimes -j`.
data class ListRuntimesReport(
@Expose val runtimes: List<SimulatorRuntimeDescriptor>
)
data class SimulatorRuntimeDescriptor(
@Expose val version: String,
// bundlePath field may not exist in the old Xcode (prior to 10.3).
@Expose val bundlePath: String? = null,
@Expose val isAvailable: Boolean? = null,
@Expose val availability: String? = null,
@Expose val name: String,
@Expose val identifier: String,
@Expose val buildversion: String
) {
/**
* Different Xcode/macOS combinations give different fields that checks
* runtime availability. This method is an umbrella for these fields.
*/
fun checkAvailability(): Boolean {
if (isAvailable == true) return true
if (availability?.contains("unavailable") == true) return false
return false
}
}
@@ -0,0 +1,25 @@
package org.jetbrains.kotlin.benchmark
import java.text.SimpleDateFormat
import java.util.*
enum class LogLevel { DEBUG, OFF }
class Logger(val level: LogLevel = LogLevel.OFF) {
private fun printStderr(message: String) {
System.err.print(message)
}
private fun currentTime(): String =
SimpleDateFormat("HH:mm:ss").format(Date())
fun log(message: String, messageLevel: LogLevel = LogLevel.DEBUG, usePrefix: Boolean = true) {
if (messageLevel == level) {
if (usePrefix) {
printStderr("[$level][${currentTime()}] $message")
} else {
printStderr("$message")
}
}
}
}
@@ -0,0 +1,278 @@
package org.jetbrains.kotlin.benchmark
import groovy.lang.Closure
import org.gradle.api.NamedDomainObjectContainer
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.Task
import org.gradle.api.artifacts.Dependency
import org.gradle.util.ConfigureUtil
import org.jetbrains.kotlin.*
import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
import org.jetbrains.kotlin.gradle.plugin.mpp.AbstractKotlinNativeTargetPreset
import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType
import org.jetbrains.kotlin.konan.target.HostManager
import javax.inject.Inject
import kotlin.reflect.KClass
internal val NamedDomainObjectContainer<KotlinSourceSet>.commonMain
get() = maybeCreate("commonMain")
internal val NamedDomainObjectContainer<KotlinSourceSet>.nativeMain
get() = maybeCreate("nativeMain")
internal val Project.nativeWarmup: Int
get() = (property("nativeWarmup") as String).toInt()
internal val Project.attempts: Int
get() = (property("attempts") as String).toInt()
internal val Project.nativeBenchResults: String
get() = property("nativeBenchResults") as String
// Gradle property to add flags to benchmarks run from command line.
internal val Project.compilerArgs: List<String>
get() = (findProperty("compilerArgs") as String?)?.split("\\s").orEmpty()
internal val Project.kotlinVersion: String
get() = property("kotlinVersion") as String
internal val Project.konanVersion: String
get() = property("konanVersion") as String
internal val Project.kotlinStdlibVersion: String
get() = property("kotlinStdlibVersion") as String
internal val Project.kotlinStdlibRepo: String
get() = property("kotlinStdlibRepo") as String
internal val Project.nativeJson: String
get() = project.property("nativeJson") as String
internal val Project.jvmJson: String
get() = project.property("jvmJson") as String
internal val Project.commonBenchmarkProperties: Map<String, Any>
get() = mapOf(
"cpu" to System.getProperty("os.arch"),
"os" to System.getProperty("os.name"),
"jdkVersion" to System.getProperty("java.version"),
"jdkVendor" to System.getProperty("java.vendor"),
"kotlinVersion" to kotlinVersion
)
open class BenchmarkExtension @Inject constructor(val project: Project) {
var applicationName: String = project.name
var commonSrcDirs: Collection<Any> = emptyList()
var nativeSrcDirs: Collection<Any> = emptyList()
var compileTasks: List<String> = emptyList()
var linkerOpts: Collection<String> = emptyList()
var compilerOpts: List<String> = emptyList()
var buildType: NativeBuildType = NativeBuildType.RELEASE
var repeatingType: BenchmarkRepeatingType = BenchmarkRepeatingType.INTERNAL
val dependencies: BenchmarkDependencies = BenchmarkDependencies()
fun dependencies(action: BenchmarkDependencies.() -> Unit) =
dependencies.action()
fun dependencies(action: Closure<*>) {
ConfigureUtil.configure(action, dependencies)
}
inner class BenchmarkDependencies {
public val sourceSets: NamedDomainObjectContainer<KotlinSourceSet>
get() = project.kotlin.sourceSets
fun project(path: String): Dependency = project.dependencies.project(mapOf("path" to path))
fun project(path: String, configuration: String): Dependency =
project.dependencies.project(mapOf("path" to path, "configuration" to configuration))
fun common(notation: Any) = sourceSets.commonMain.dependencies {
implementation(notation)
}
fun native(notation: Any) = sourceSets.nativeMain.dependencies {
implementation(notation)
}
}
}
/**
* A plugin configuring a benchmark Kotlin/Native project.
*/
abstract class BenchmarkingPlugin: Plugin<Project> {
protected abstract val Project.nativeExecutable: String
protected abstract val Project.nativeLinkTask: Task
protected abstract val Project.benchmark: BenchmarkExtension
protected abstract val benchmarkExtensionName: String
protected abstract val benchmarkExtensionClass: KClass<*>
protected val mingwPath: String = System.getenv("MINGW64_DIR") ?: "c:/msys64/mingw64"
protected open fun Project.determinePreset(): AbstractKotlinNativeTargetPreset<*> =
defaultHostPreset(this).also { preset ->
logger.quiet("$project has been configured for ${preset.name} platform.")
} as AbstractKotlinNativeTargetPreset<*>
protected abstract fun NamedDomainObjectContainer<KotlinSourceSet>.configureSources(project: Project)
protected open fun NamedDomainObjectContainer<KotlinSourceSet>.additionalConfigurations(project: Project) {}
protected open fun Project.configureSourceSets(kotlinVersion: String) {
with(kotlin.sourceSets) {
commonMain.dependencies {
implementation("org.jetbrains.kotlin:kotlin-stdlib-common:$kotlinStdlibVersion")
}
project.configurations.getByName(nativeMain.implementationConfigurationName).apply {
// Exclude dependencies already included into K/N distribution (aka endorsed libraries).
exclude(mapOf("module" to "kotlinx.cli"))
}
repositories.maven {
it.setUrl(kotlinStdlibRepo)
}
additionalConfigurations(this@configureSourceSets)
// Add sources specified by a user in the benchmark DSL.
afterEvaluate {
configureSources(project)
}
}
}
protected open fun KotlinNativeTarget.configureNativeOutput(project: Project) {
binaries.executable(NATIVE_EXECUTABLE_NAME, listOf(project.benchmark.buildType)) {
if (HostManager.hostIsMingw) {
linkerOpts.add("-L${mingwPath}/lib")
}
runTask!!.apply {
group = ""
enabled = false
}
// Specify settings configured by a user in the benchmark extension.
project.afterEvaluate {
linkerOpts.addAll(project.benchmark.linkerOpts)
freeCompilerArgs = project.benchmark.compilerOpts + project.compilerArgs
}
}
}
protected fun Project.configureNativeTarget(hostPreset: AbstractKotlinNativeTargetPreset<*>) {
kotlin.targetFromPreset(hostPreset, NATIVE_TARGET_NAME) {
compilations.getByName("main").kotlinOptions.freeCompilerArgs = benchmark.compilerOpts + project.compilerArgs
compilations.getByName("main").enableEndorsedLibs = true
configureNativeOutput(this@configureNativeTarget)
}
}
protected open fun configureMPPExtension(project: Project) {
project.configureSourceSets(project.kotlinVersion)
project.configureNativeTarget(project.determinePreset())
}
protected open fun Project.configureNativeTask(nativeTarget: KotlinNativeTarget): Task {
val konanRun = createRunTask(this, "konanRun", nativeLinkTask,
nativeExecutable, buildDir.resolve(nativeBenchResults).absolutePath).apply {
group = BENCHMARKING_GROUP
description = "Runs the benchmark for Kotlin/Native."
}
afterEvaluate {
val task = konanRun as RunKotlinNativeTask
task.args("-p", "${benchmark.applicationName}::")
task.warmupCount = nativeWarmup
task.repeatCount = attempts
task.repeatingType = benchmark.repeatingType
}
return konanRun
}
protected abstract fun Project.configureJvmTask(): Task
protected fun compilerFlagsFromBinary(project: Project): List<String> {
val result = mutableListOf<String>()
if (project.benchmark.buildType.optimized) {
result.add("-opt")
}
if (project.benchmark.buildType.debuggable) {
result.add("-g")
}
return result
}
protected open fun getCompilerFlags(project: Project, nativeTarget: KotlinNativeTarget) =
compilerFlagsFromBinary(project) + nativeTarget.compilations.main.kotlinOptions.freeCompilerArgs.map { "\"$it\"" }
protected open fun Project.collectCodeSize(applicationName: String) =
getCodeSizeBenchmark(applicationName, nativeExecutable)
protected open fun Project.configureKonanJsonTask(nativeTarget: KotlinNativeTarget): Task {
return tasks.create("konanJsonReport") {
it.group = BENCHMARKING_GROUP
it.description = "Builds the benchmarking report for Kotlin/Native."
it.doLast {
val applicationName = benchmark.applicationName
val benchContents = buildDir.resolve(nativeBenchResults).readText()
val nativeCompileTime = if (benchmark.compileTasks.isEmpty()) getNativeCompileTime(project, applicationName)
else getNativeCompileTime(project, applicationName, benchmark.compileTasks)
val properties = commonBenchmarkProperties + mapOf(
"type" to "native",
"compilerVersion" to konanVersion,
"flags" to getCompilerFlags(project, nativeTarget).sorted(),
"benchmarks" to benchContents,
"compileTime" to listOf(nativeCompileTime),
"codeSize" to collectCodeSize(applicationName)
)
val output = createJsonReport(properties)
buildDir.resolve(nativeJson).writeText(output)
}
}
}
protected abstract fun Project.configureJvmJsonTask(jvmRun: Task): Task
protected open fun Project.configureExtraTasks() {}
private fun Project.configureTasks() {
val nativeTarget = kotlin.targets.getByName(NATIVE_TARGET_NAME) as KotlinNativeTarget
configureExtraTasks()
// Native run task.
configureNativeTask(nativeTarget)
// JVM run task.
val jvmRun = configureJvmTask()
// Native report task.
configureKonanJsonTask(nativeTarget)
// JVM report task.
configureJvmJsonTask(jvmRun)
}
override fun apply(target: Project) = with(target) {
pluginManager.apply("kotlin-multiplatform")
// Use Kotlin compiler version specified by the project property.
dependencies.add("kotlinCompilerClasspath", "org.jetbrains.kotlin:kotlin-compiler-embeddable:$kotlinVersion")
addTimeListener(this)
extensions.create(benchmarkExtensionName, benchmarkExtensionClass.java, this)
configureMPPExtension(this)
configureTasks()
}
companion object {
const val NATIVE_TARGET_NAME = "native"
const val NATIVE_EXECUTABLE_NAME = "benchmark"
const val BENCHMARKING_GROUP = "benchmarking"
}
}
@@ -0,0 +1,148 @@
package org.jetbrains.kotlin.benchmark
import groovy.lang.Closure
import org.gradle.api.*
import org.gradle.api.tasks.Delete
import org.gradle.api.tasks.Exec
import org.gradle.util.ConfigureUtil
import org.jetbrains.kotlin.*
import javax.inject.Inject
class BuildStep (private val _name: String): Named {
override fun getName(): String = _name
lateinit var command: List<String>
fun command(vararg command: String) {
this.command = command.toList()
}
}
class BuildStepContainer(project: Project): NamedDomainObjectContainer<BuildStep> by project.container(BuildStep::class.java) {
fun step(name: String, configure: Action<BuildStep>) =
maybeCreate(name).apply { configure.execute(this) }
fun step(name: String, configure: Closure<Unit>) =
step(name, ConfigureUtil.configureUsing(configure))
}
open class CompileBenchmarkExtension @Inject constructor(val project: Project) {
var applicationName = project.name
var repeatNumber: Int = 1
var buildSteps: BuildStepContainer = BuildStepContainer(project)
var compilerOpts: List<String> = emptyList()
fun buildSteps(configure: Action<BuildStepContainer>): Unit = buildSteps.let { configure.execute(it) }
fun buildSteps(configure: Closure<Unit>): Unit = buildSteps(ConfigureUtil.configureUsing(configure))
}
open class CompileBenchmarkingPlugin : Plugin<Project> {
private val exitCodes: MutableMap<String, Int> = mutableMapOf()
private fun Project.configureUtilityTasks() {
tasks.create("configureBuild") {
it.doLast { mkdir(buildDir) }
}
tasks.create("clean", Delete::class.java) {
it.delete(buildDir)
}
}
private fun Project.configureKonanRun(
benchmarkExtension: CompileBenchmarkExtension
): Unit = with(benchmarkExtension) {
// Aggregate task.
val konanRun = tasks.create("konanRun") { task ->
task.dependsOn("configureBuild")
task.group = BenchmarkingPlugin.BENCHMARKING_GROUP
task.description = "Runs the compile only benchmark for Kotlin/Native."
}
// Compile tasks.
afterEvaluate {
for (number in 1..repeatNumber) {
buildSteps.forEach { step ->
val taskName = step.name
tasks.create("$taskName$number", Exec::class.java).apply {
commandLine(step.command)
isIgnoreExitValue = true
konanRun.dependsOn(this)
doLast {
exitCodes[name] = execResult!!.exitValue
}
}
}
}
}
// Report task.
tasks.create("konanJsonReport").apply {
group = BenchmarkingPlugin.BENCHMARKING_GROUP
description = "Builds the benchmarking report for Kotlin/Native."
doLast {
val nativeCompileTime = getCompileBenchmarkTime(
project,
applicationName,
buildSteps.names,
repeatNumber,
exitCodes
)
val nativeExecutable = buildDir.resolve("program${getNativeProgramExtension()}")
val properties = commonBenchmarkProperties + mapOf(
"type" to "native",
"compilerVersion" to konanVersion,
"benchmarks" to "[]",
"flags" to getCompilerFlags(benchmarkExtension).sorted(),
"compileTime" to nativeCompileTime,
"codeSize" to getCodeSizeBenchmark(applicationName, nativeExecutable.absolutePath)
)
val output = createJsonReport(properties)
buildDir.resolve(nativeJson).writeText(output)
}
konanRun.finalizedBy(this)
}
}
private fun getCompilerFlags(benchmarkExtension: CompileBenchmarkExtension) =
benchmarkExtension.compilerOpts
private fun Project.configureJvmRun(
benchmarkExtension: CompileBenchmarkExtension
) {
val jvmRun = tasks.create("jvmRun") {
it.group = BenchmarkingPlugin.BENCHMARKING_GROUP
it.description = "Runs the compile only benchmark for Kotlin/JVM."
it.doLast { println("JVM run isn't supported") }
}
tasks.create("jvmJsonReport") {
it.group = BenchmarkingPlugin.BENCHMARKING_GROUP
it.description = "Builds the benchmarking report for Kotlin/Native."
it.doLast { println("JVM run isn't supported") }
jvmRun.finalizedBy(it)
}
}
override fun apply(target: Project): Unit = with(target) {
addTimeListener(this)
val benchmarkExtension = extensions.create(
COMPILE_BENCHMARK_EXTENSION_NAME,
CompileBenchmarkExtension::class.java,
this
)
// Create tasks.
configureUtilityTasks()
configureKonanRun(benchmarkExtension)
configureJvmRun(benchmarkExtension)
}
companion object {
const val COMPILE_BENCHMARK_EXTENSION_NAME = "compileBenchmark"
}
}
@@ -0,0 +1,146 @@
package org.jetbrains.kotlin.benchmark
import org.gradle.jvm.tasks.Jar
import org.gradle.api.NamedDomainObjectContainer
import org.gradle.api.Project
import org.gradle.api.Task
import org.jetbrains.kotlin.*
import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
import org.jetbrains.kotlin.gradle.plugin.mpp.Executable
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
import org.jetbrains.kotlin.konan.target.HostManager
import javax.inject.Inject
import kotlin.reflect.KClass
private val NamedDomainObjectContainer<KotlinSourceSet>.jvmMain
get() = maybeCreate("jvmMain")
private val Project.jvmWarmup: Int
get() = (property("jvmWarmup") as String).toInt()
private val Project.jvmBenchResults: String
get() = property("jvmBenchResults") as String
open class KotlinNativeBenchmarkExtension @Inject constructor(project: Project) : BenchmarkExtension(project) {
var jvmSrcDirs: Collection<Any> = emptyList()
var mingwSrcDirs: Collection<Any> = emptyList()
var posixSrcDirs: Collection<Any> = emptyList()
fun BenchmarkExtension.BenchmarkDependencies.jvm(notation: Any) = sourceSets.jvmMain.dependencies {
implementation(notation)
}
}
/**
* A plugin configuring a benchmark Kotlin/Native project.
*/
open class KotlinNativeBenchmarkingPlugin: BenchmarkingPlugin() {
override fun Project.configureJvmJsonTask(jvmRun: Task): Task {
return tasks.create("jvmJsonReport") {
it.group = BENCHMARKING_GROUP
it.description = "Builds the benchmarking report for Kotlin/JVM."
it.doLast {
val applicationName = benchmark.applicationName
val jarPath = (tasks.getByName("jvmJar") as Jar).archiveFile.get().asFile
val jvmCompileTime = getJvmCompileTime(project, applicationName)
val benchContents = buildDir.resolve(jvmBenchResults).readText()
val properties: Map<String, Any> = commonBenchmarkProperties + mapOf(
"type" to "jvm",
"compilerVersion" to kotlinVersion,
"benchmarks" to benchContents,
"compileTime" to listOf(jvmCompileTime),
"codeSize" to getCodeSizeBenchmark(applicationName, jarPath.absolutePath)
)
val output = createJsonReport(properties)
buildDir.resolve(jvmJson).writeText(output)
}
jvmRun.finalizedBy(it)
}
}
override fun Project.configureJvmTask(): Task {
return tasks.create("jvmRun", RunJvmTask::class.java) { task ->
task.dependsOn("jvmJar")
val mainCompilation = kotlin.jvm().compilations.getByName("main")
val runtimeDependencies = configurations.getByName(mainCompilation.runtimeDependencyConfigurationName)
task.classpath(files(mainCompilation.output.allOutputs, runtimeDependencies))
task.main = "MainKt"
task.group = BENCHMARKING_GROUP
task.description = "Runs the benchmark for Kotlin/JVM."
// Specify settings configured by a user in the benchmark extension.
afterEvaluate {
task.args("-p", "${benchmark.applicationName}::")
task.warmupCount = jvmWarmup
task.repeatCount = attempts
task.outputFileName = buildDir.resolve(jvmBenchResults).absolutePath
task.repeatingType = benchmark.repeatingType
}
}
}
override val benchmarkExtensionClass: KClass<*>
get() = KotlinNativeBenchmarkExtension::class
override val Project.benchmark: KotlinNativeBenchmarkExtension
get() = extensions.getByName(benchmarkExtensionName) as KotlinNativeBenchmarkExtension
override val benchmarkExtensionName: String = "benchmark"
private val Project.nativeBinary: Executable
get() = (kotlin.targets.getByName(NATIVE_TARGET_NAME) as KotlinNativeTarget)
.binaries.getExecutable(NATIVE_EXECUTABLE_NAME, benchmark.buildType)
override val Project.nativeExecutable: String
get() = nativeBinary.outputFile.absolutePath
override val Project.nativeLinkTask: Task
get() = nativeBinary.linkTask
override fun configureMPPExtension(project: Project) {
super.configureMPPExtension(project)
project.configureJVMTarget()
}
override fun getCompilerFlags(project: Project, nativeTarget: KotlinNativeTarget) =
super.getCompilerFlags(project, nativeTarget) + project.nativeBinary.freeCompilerArgs.map { "\"$it\"" }
override fun NamedDomainObjectContainer<KotlinSourceSet>.configureSources(project: Project) {
project.benchmark.let {
commonMain.kotlin.srcDirs(*it.commonSrcDirs.toTypedArray())
if (HostManager.hostIsMingw) {
nativeMain.kotlin.srcDirs(*(it.nativeSrcDirs + it.mingwSrcDirs).toTypedArray())
} else {
nativeMain.kotlin.srcDirs(*(it.nativeSrcDirs + it.posixSrcDirs).toTypedArray())
}
jvmMain.kotlin.srcDirs(*it.jvmSrcDirs.toTypedArray())
}
}
override fun NamedDomainObjectContainer<KotlinSourceSet>.additionalConfigurations(project: Project) {
jvmMain.dependencies {
implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8:${project.kotlinStdlibVersion}")
}
}
private fun Project.configureJVMTarget() {
kotlin.jvm {
compilations.all {
it.compileKotlinTask.kotlinOptions {
jvmTarget = "1.8"
suppressWarnings = true
freeCompilerArgs = project.benchmark.compilerOpts + project.compilerArgs
}
}
}
}
companion object {
const val BENCHMARK_EXTENSION_NAME = "benchmark"
}
}
@@ -0,0 +1,107 @@
package org.jetbrains.kotlin.benchmark
import org.gradle.api.NamedDomainObjectContainer
import org.gradle.api.Project
import org.gradle.api.Task
import org.jetbrains.kotlin.*
import org.jetbrains.kotlin.gradle.plugin.KotlinSourceSet
import org.jetbrains.kotlin.gradle.plugin.mpp.Framework
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinNativeTarget
import org.jetbrains.kotlin.gradle.plugin.mpp.AbstractKotlinNativeTargetPreset
import org.jetbrains.kotlin.gradle.plugin.mpp.NativeBuildType
import java.io.File
import javax.inject.Inject
import java.nio.file.Paths
import kotlin.reflect.KClass
enum class CodeSizeEntity { FRAMEWORK, EXECUTABLE }
open class SwiftBenchmarkExtension @Inject constructor(project: Project) : BenchmarkExtension(project) {
var swiftSources: List<String> = emptyList()
var useCodeSize: CodeSizeEntity = CodeSizeEntity.FRAMEWORK // use as code size metric framework size or executable
}
/**
* A plugin configuring a benchmark Kotlin/Native project.
*/
open class SwiftBenchmarkingPlugin : BenchmarkingPlugin() {
override fun Project.configureJvmJsonTask(jvmRun: Task): Task {
return tasks.create("jvmJsonReport") {
logger.info("JVM run is unsupported")
jvmRun.finalizedBy(it)
}
}
override fun Project.configureJvmTask(): Task {
return tasks.create("jvmRun") { task ->
task.doLast {
logger.info("JVM run is unsupported")
}
}
}
override val benchmarkExtensionClass: KClass<*>
get() = SwiftBenchmarkExtension::class
override val Project.benchmark: SwiftBenchmarkExtension
get() = extensions.getByName(benchmarkExtensionName) as SwiftBenchmarkExtension
override val benchmarkExtensionName: String = "swiftBenchmark"
override val Project.nativeExecutable: String
get() = Paths.get(buildDir.absolutePath, benchmark.applicationName).toString()
override val Project.nativeLinkTask: Task
get() = tasks.getByName("buildSwift")
private lateinit var framework: Framework
val nativeFrameworkName = "benchmark"
override fun NamedDomainObjectContainer<KotlinSourceSet>.configureSources(project: Project) {
project.benchmark.let {
commonMain.kotlin.srcDirs(*it.commonSrcDirs.toTypedArray())
nativeMain.kotlin.srcDirs(*(it.nativeSrcDirs).toTypedArray())
}
}
override fun Project.determinePreset(): AbstractKotlinNativeTargetPreset<*> = kotlin.presets.macosX64 as AbstractKotlinNativeTargetPreset<*>
override fun KotlinNativeTarget.configureNativeOutput(project: Project) {
binaries.framework(nativeFrameworkName, listOf(project.benchmark.buildType)) {
// Specify settings configured by a user in the benchmark extension.
project.afterEvaluate {
linkerOpts.addAll(project.benchmark.linkerOpts)
}
}
}
override fun Project.configureExtraTasks() {
val nativeTarget = kotlin.targets.getByName(NATIVE_TARGET_NAME) as KotlinNativeTarget
// Build executable from swift code.
framework = nativeTarget.binaries.getFramework(nativeFrameworkName, benchmark.buildType)
val buildSwift = tasks.create("buildSwift") { task ->
task.dependsOn(framework.linkTaskName)
task.doLast {
val frameworkParentDirPath = framework.outputDirectory.absolutePath
val options = listOf("-O", "-wmo", "-Xlinker", "-rpath", "-Xlinker", frameworkParentDirPath, "-F", frameworkParentDirPath)
compileSwift(project, nativeTarget.konanTarget, benchmark.swiftSources, options,
Paths.get(buildDir.absolutePath, benchmark.applicationName), false)
}
}
}
override fun Project.collectCodeSize(applicationName: String) =
getCodeSizeBenchmark(applicationName,
if (benchmark.useCodeSize == CodeSizeEntity.FRAMEWORK)
File("${framework.outputFile.absolutePath}/$nativeFrameworkName").canonicalPath
else
nativeExecutable
)
override fun getCompilerFlags(project: Project, nativeTarget: KotlinNativeTarget) =
if (project.benchmark.useCodeSize == CodeSizeEntity.FRAMEWORK) {
super.getCompilerFlags(project, nativeTarget) + framework.freeCompilerArgs.map { "\"$it\"" }
} else {
listOf("-O", "-wmo")
}
}
@@ -0,0 +1,134 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.bitcode
import org.gradle.api.DefaultTask
import org.gradle.api.file.FileCollection
import org.gradle.api.tasks.*
import org.jetbrains.kotlin.ExecClang
import org.jetbrains.kotlin.konan.target.Family
import org.jetbrains.kotlin.konan.target.HostManager
import org.jetbrains.kotlin.konan.target.KonanTarget
import java.io.File
import javax.inject.Inject
open class CompileToBitcode @Inject constructor(
val srcRoot: File,
val folderName: String,
val target: String,
val outputGroup: String
) : DefaultTask() {
enum class Language {
C, CPP
}
// Compiler args are part of compilerFlags so we don't register them as an input.
val compilerArgs = mutableListOf<String>()
@Input
val linkerArgs = mutableListOf<String>()
var excludeFiles: List<String> = listOf(
"**/*Test.cpp",
"**/*Test.mm",
)
var includeFiles: List<String> = listOf(
"**/*.cpp",
"**/*.mm"
)
// Source files and headers are registered as inputs by the `inputFiles` and `headers` properties.
var srcDirs: FileCollection = project.files(srcRoot.resolve("cpp"))
var headersDirs: FileCollection = project.files(srcRoot.resolve("headers"))
@Input
var skipLinkagePhase = false
@Input
var language = Language.CPP
private val targetDir by lazy { project.buildDir.resolve("bitcode/$outputGroup/$target") }
val objDir by lazy { File(targetDir, folderName) }
private val KonanTarget.isMINGW
get() = this.family == Family.MINGW
val executable
get() = when (language) {
Language.C -> "clang"
Language.CPP -> "clang++"
}
@get:Input
val compilerFlags: List<String>
get() {
val commonFlags = listOf("-c", "-emit-llvm") + headersDirs.map { "-I$it" }
val languageFlags = when (language) {
Language.C ->
// Used flags provided by original build of allocator C code.
listOf("-std=gnu11", "-O3", "-Wall", "-Wextra", "-Werror")
Language.CPP ->
listOfNotNull("-std=c++14", "-Werror", "-O2",
"-Wall", "-Wextra",
"-Wno-unused-parameter", // False positives with polymorphic functions.
"-Wno-unused-function", // TODO: Enable this warning when we have C++ runtime tests.
"-fPIC".takeIf { !HostManager().targetByName(target).isMINGW })
}
return commonFlags + languageFlags + compilerArgs
}
@get:SkipWhenEmpty
@get:InputFiles
val inputFiles: Iterable<File>
get() {
return srcDirs.flatMap { srcDir ->
project.fileTree(srcDir) {
it.include(includeFiles)
it.exclude(excludeFiles)
}.files
}
}
@get:InputFiles
protected val headers: Iterable<File>
get() {
return headersDirs.files.flatMap { dir ->
project.fileTree(dir) {
val includePatterns = when (language) {
Language.C -> arrayOf("**/.h")
Language.CPP -> arrayOf("**/*.h", "**/*.hpp")
}
it.include(*includePatterns)
}.files
}
}
@OutputFile
val outFile = File(targetDir, "${folderName}.bc")
@TaskAction
fun compile() {
objDir.mkdirs()
val plugin = project.convention.getPlugin(ExecClang::class.java)
plugin.execKonanClang(target) {
it.workingDir = objDir
it.executable = executable
it.args = compilerFlags + inputFiles.map { it.absolutePath }
}
if (!skipLinkagePhase) {
project.exec {
val llvmDir = project.findProperty("llvmDir")
it.executable = "$llvmDir/bin/llvm-link"
it.args = listOf("-o", outFile.absolutePath) + linkerArgs +
project.fileTree(objDir) {
it.include("**/*.bc")
}.files.map { it.absolutePath }
}
}
}
}
@@ -0,0 +1,64 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.bitcode
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.plugins.BasePlugin
import org.jetbrains.kotlin.createCompilationDatabasesFromCompileToBitcodeTasks
import java.io.File
import javax.inject.Inject
/**
* A plugin creating extensions to compile
*/
open class CompileToBitcodePlugin: Plugin<Project> {
override fun apply(target: Project) = with(target) {
extensions.create(EXTENSION_NAME, CompileToBitcodeExtension::class.java, target)
afterEvaluate {
// TODO: Support providers (https://docs.gradle.org/current/userguide/lazy_configuration.html)
// in database tasks and create them along with corresponding compile tasks (not in afterEvaluate).
createCompilationDatabasesFromCompileToBitcodeTasks(project, COMPILATION_DATABASE_TASK_NAME)
}
}
companion object {
const val EXTENSION_NAME = "bitcode"
const val COMPILATION_DATABASE_TASK_NAME = "CompilationDatabase"
}
}
open class CompileToBitcodeExtension @Inject constructor(val project: Project) {
private val targetList = with(project) {
provider { rootProject.property("targetList") as List<String> } // TODO: Can we make it better?
}
fun create(
name: String,
srcDir: File = project.file("src/$name"),
outputGroup: String = "main",
configurationBlock: CompileToBitcode.() -> Unit = {}
) {
targetList.get().forEach { targetName ->
project.tasks.register(
"${targetName}${name.snakeCaseToCamelCase().capitalize()}",
CompileToBitcode::class.java,
srcDir, name, targetName, outputGroup
).configure {
it.group = BasePlugin.BUILD_GROUP
it.description = "Compiles '$name' to bitcode for $targetName"
it.configurationBlock()
}
}
}
companion object {
private fun String.snakeCaseToCamelCase() =
split('_').joinToString(separator = "") { it.capitalize() }
}
}
@@ -0,0 +1,144 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin
// This is almost a full copy of kotlin/compiler/tests-common/tests/org/jetbrains/kotlin/coroutineTestUtil.kt
// TODO: get it automatically as a dependency
fun createTextForHelpers(isReleaseCoroutines: Boolean): String {
val coroutinesPackage = "kotlin.coroutines"
val emptyContinuationBody =
if (isReleaseCoroutines)
"""
|override fun resumeWith(result: Result<Any?>) {
| result.getOrThrow()
|}
""".trimMargin()
else
"""
|override fun resume(data: Any?) {}
|override fun resumeWithException(exception: Throwable) { throw exception }
""".trimMargin()
val handleResultContinuationBody =
if (isReleaseCoroutines)
"""
|override fun resumeWith(result: Result<T>) {
| x(result.getOrThrow())
|}
""".trimMargin()
else
"""
|override fun resumeWithException(exception: Throwable) {
| throw exception
|}
|
|override fun resume(data: T) = x(data)
""".trimMargin()
val handleExceptionContinuationBody =
if (isReleaseCoroutines)
"""
|override fun resumeWith(result: Result<Any?>) {
| result.exceptionOrNull()?.let(x)
|}
""".trimMargin()
else
"""
|override fun resumeWithException(exception: Throwable) {
| x(exception)
|}
|
|override fun resume(data: Any?) {}
""".trimMargin()
val continuationAdapterBody =
if (isReleaseCoroutines)
"""
|override fun resumeWith(result: Result<T>) {
| if (result.isSuccess) {
| resume(result.getOrThrow())
| } else {
| resumeWithException(result.exceptionOrNull()!!)
| }
|}
|
|abstract fun resumeWithException(exception: Throwable)
|abstract fun resume(value: T)
""".trimMargin()
else
""
return """
|package helpers
|import $coroutinesPackage.*
|
|fun <T> handleResultContinuation(x: (T) -> Unit): Continuation<T> = object: Continuation<T> {
| override val context = EmptyCoroutineContext
| $handleResultContinuationBody
|}
|
|
|fun handleExceptionContinuation(x: (Throwable) -> Unit): Continuation<Any?> = object: Continuation<Any?> {
| override val context = EmptyCoroutineContext
| $handleExceptionContinuationBody
|}
|
|open class EmptyContinuation(override val context: CoroutineContext = EmptyCoroutineContext) : Continuation<Any?> {
| companion object : EmptyContinuation()
| $emptyContinuationBody
|}
|
|abstract class ContinuationAdapter<in T> : Continuation<T> {
| override val context: CoroutineContext = EmptyCoroutineContext
| $continuationAdapterBody
|}
|class StateMachineCheckerClass {
| private var counter = 0
| var finished = false
|
| var proceed: () -> Unit = {}
|
| fun reset() {
| counter = 0
| finished = false
| proceed = {}
| }
|
| suspend fun suspendHere() = suspendCoroutine<Unit> { c ->
| counter++
| proceed = { c.resume(Unit) }
| }
|
| fun check(numberOfSuspensions: Int, checkFinished: Boolean = true) {
| for (i in 1..numberOfSuspensions) {
| if (counter != i) error("Wrong state-machine generated: suspendHere called should be called exactly once in one state. Expected " + i + ", got " + counter)
| proceed()
| }
| if (counter != numberOfSuspensions)
| error("Wrong state-machine generated: suspendHere called should be called exactly once in one state. Expected " + numberOfSuspensions + ", got " + counter)
| if (finished) error("Wrong state-machine generated: it is finished early")
| proceed()
| if (checkFinished && !finished) error("Wrong state-machine generated: it is not finished yet")
| }
|}
|val StateMachineChecker = StateMachineCheckerClass()
|object CheckStateMachineContinuation: ContinuationAdapter<Unit>() {
| override val context: CoroutineContext
| get() = EmptyCoroutineContext
|
| override fun resume(value: Unit) {
| StateMachineChecker.proceed = {
| StateMachineChecker.finished = true
| }
| }
|
| override fun resumeWithException(exception: Throwable) {
| throw exception
| }
|}
""".trimMargin()
}
@@ -0,0 +1,37 @@
package org.jetbrains.kotlin
import java.io.File
fun genTestKT39548(file: File) {
val longName = StringBuilder().apply {
repeat(10_000_000) {
append('a')
}
}
val text = """
import kotlin.test.*
fun $longName(): Int = 42
fun <T> same(value: T): T = value
val globalInt1: Int = same(1)
val globalStringA: String = same("a")
@ThreadLocal val threadLocalInt2: Int = same(2)
@ThreadLocal val threadLocalStringB: String = same("b")
fun main() {
// Ensure function don't get DCEd:
val resultOfFunctionWithLongName = $longName()
assertEquals(42, resultOfFunctionWithLongName)
// Check that top-level initializers did run as expected:
assertEquals(1, globalInt1)
assertEquals("a", globalStringA)
assertEquals(2, threadLocalInt2)
assertEquals("b", threadLocalStringB)
}
""".trimIndent()
file.parentFile.mkdirs()
file.writeText(text)
}
@@ -0,0 +1,195 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.klib.metadata
import kotlinx.metadata.*
import kotlinx.metadata.klib.*
/**
* Structural comparison of Km* metadata.
* [configuration] allows to tune comparison process.
*
*/
internal class KmComparator(private val configuration: ComparisonConfig) {
fun compare(kmClass1: KmClass, kmClass2: KmClass): MetadataCompareResult = serialComparator(
compare(KmClass::name, ::compare) to "Different names: ${kmClass1.name}, ${kmClass2.name}",
::compareClassFlags to "Different flags for ${kmClass1.name}",
compare(KmClass::constructors, compareLists(::compare)) to "Constructors mismatch for ${kmClass1.name}",
compare(KmClass::properties, compareLists(::compare, KmProperty::mangle)) to "Properties mismatch for ${kmClass1.name}",
compare(KmClass::functions, compareLists(::compare, KmFunction::mangle)) to "Functions mismatch for ${kmClass1.name}",
)(kmClass1, kmClass2)
fun compare(typealias1: KmTypeAlias, typealias2: KmTypeAlias): MetadataCompareResult = serialComparator(
compare(KmTypeAlias::name, ::compare) to "Different names",
compare(KmTypeAlias::underlyingType, ::compareTypes) to "Underlying types mismatch",
compare(KmTypeAlias::expandedType, ::compareTypes) to "Expanded types mismatch",
compare(KmTypeAlias::typeParameters, compareLists(::compare)) to "Type parameters mismatch"
)(typealias1, typealias2)
fun compare(function1: KmFunction, function2: KmFunction): MetadataCompareResult = serialComparator(
compare(KmFunction::name, ::compare) to "Different names",
compare(KmFunction::returnType, ::compareTypes) to "Return type mismatch",
compare(KmFunction::valueParameters, compareLists(::compare)) to "Value parameters mismatch",
::compareFunctionFlags to "Flags mismatch"
)(function1, function2)
fun compare(property1: KmProperty, property2: KmProperty): MetadataCompareResult = serialComparator(
compare(KmProperty::name, ::compare) to "Different names",
compare(KmProperty::returnType, ::compareTypes) to "Return type mismatch",
::comparePropertyFlags to "Flags mismatch",
(compare(KmProperty::getterFlags, ::comparePropertyAccessorFlags) to "Getter flags mismatch")
.takeIf { Flag.Property.HAS_GETTER(property1.flags) && Flag.Property.HAS_GETTER(property2.flags) },
(compare(KmProperty::setterFlags, ::comparePropertyAccessorFlags) to "Setter flags mismatch")
.takeIf { Flag.Property.HAS_SETTER(property1.flags) && Flag.Property.HAS_SETTER(property2.flags) }
)(property1, property2)
private fun compare(entry1: KlibEnumEntry, entry2: KlibEnumEntry): MetadataCompareResult = serialComparator(
compare(KlibEnumEntry::annotations, compareLists(::compare)) to "Different annotations",
compare(KlibEnumEntry::name, ::compare) to "Different names",
compare(KlibEnumEntry::ordinal, compareNullable(::compare)) to "Different ordinals"
)(entry1, entry2)
private fun checkFlag(flag: Flag, flagName: String? = null): (Flags, Flags) -> MetadataCompareResult = { f1, f2 ->
when {
flag(f1) != flag(f2) -> Fail("Flags mismatch: ${flag(f1)}, ${flag(f2)} for $flagName")
else -> Ok
}
}
private fun compare(value1: Int, value2: Int): MetadataCompareResult = when {
value1 == value2 -> Ok
else -> Fail("$value1 != $value2")
}
private fun compare(string1: String, string2: String): MetadataCompareResult = when {
string1 == string2 -> Ok
else -> Fail("$string1 != $string2")
}
private fun compareFunctionFlags(function1: KmFunction, function2: KmFunction): MetadataCompareResult =
serialComparator(
checkFlag(Flag.Function.IS_EXTERNAL, "IS_EXTERNAL"),
checkFlag(Flag.Function.IS_DECLARATION, "IS_DECLARATION"),
::compareVisibilityFlags,
::compareModalityFlags
)(function1.flags, function2.flags)
private fun comparePropertyFlags(property1: KmProperty, property2: KmProperty): MetadataCompareResult =
serialComparator(
checkFlag(Flag.Property.IS_CONST, "IS_CONST"),
checkFlag(Flag.Property.HAS_SETTER, "HAS_SETTER"),
checkFlag(Flag.Property.HAS_GETTER, "HAS_GETTER"),
checkFlag(Flag.Property.IS_VAR, "IS_VAR"),
checkFlag(Flag.Property.HAS_CONSTANT, "HAS_CONSTANT"),
checkFlag(Flag.Property.IS_DECLARATION, "IS_DECLARATION"),
checkFlag(Flag.Property.IS_EXTERNAL, "IS_EXTERNAL")
)(property1.flags, property2.flags)
private fun comparePropertyAccessorFlags(flags1: Flags, flags2: Flags): MetadataCompareResult = serialComparator(
checkFlag(Flag.PropertyAccessor.IS_NOT_DEFAULT, "IS_NOT_DEFAULT"),
checkFlag(Flag.PropertyAccessor.IS_INLINE, "IS_INLINE"),
checkFlag(Flag.PropertyAccessor.IS_EXTERNAL, "IS_EXTERNAL"),
::compareVisibilityFlags,
::compareModalityFlags
)(flags1, flags2)
private fun compareClassFlags(class1: KmClass, class2: KmClass): MetadataCompareResult = serialComparator(
checkFlag(Flag.Class.IS_CLASS, "IS_CLASS"),
checkFlag(Flag.Class.IS_COMPANION_OBJECT, "IS_COMPANION_OBJECT"),
checkFlag(Flag.Class.IS_ENUM_CLASS, "IS_ENUM_CLASS"),
checkFlag(Flag.Class.IS_ENUM_ENTRY, "IS_ENUM_ENTRY"),
checkFlag(Flag.Class.IS_OBJECT, "IS_OBJECT"),
checkFlag(Flag.IS_FINAL, "IS_FINAL"),
checkFlag(Flag.IS_OPEN, "IS_OPEN"),
checkFlag(Flag.HAS_ANNOTATIONS, "HAS_ANNOTATIONS"),
::compareVisibilityFlags,
::compareModalityFlags
)(class1.flags, class2.flags)
private fun compareVisibilityFlags(flags1: Flags, flags2: Flags): MetadataCompareResult = serialComparator(
checkFlag(Flag.IS_PUBLIC, "IS_PUBLIC"),
checkFlag(Flag.IS_PRIVATE_TO_THIS, "IS_PRIVATE_TO_THIS"),
checkFlag(Flag.IS_PRIVATE, "IS_PRIVATE"),
checkFlag(Flag.IS_PROTECTED, "IS_PROTECTED"),
checkFlag(Flag.IS_INTERNAL, "IS_INTERNAL")
)(flags1, flags2)
private fun compareModalityFlags(flags1: Flags, flags2: Flags): MetadataCompareResult = serialComparator(
checkFlag(Flag.IS_FINAL, "IS_FINAL"),
checkFlag(Flag.IS_ABSTRACT, "IS_ABSTRACT"),
checkFlag(Flag.IS_OPEN, "IS_OPEN"),
checkFlag(Flag.IS_SEALED, "IS_SEALED")
)(flags1, flags2)
private fun compare(annotation1: KmAnnotation, annotation2: KmAnnotation): MetadataCompareResult = when {
annotation1.className != annotation2.className -> Fail("${annotation1.className} != ${annotation2.className}")
// TODO: compare values
else -> Ok
}
private fun compare(p1: KmValueParameter, p2: KmValueParameter): MetadataCompareResult = serialComparator(
compare(KmValueParameter::name, ::compare) to "Different names",
compare(KmValueParameter::type, compareNullable(::compareTypes)) to "Type mismatch",
compare(KmValueParameter::annotations, compareLists(::compare)) to "Annotations mismatch"
)(p1, p2)
private fun compareTypeFlags(flags1: Flags, flags2: Flags): MetadataCompareResult = serialComparator(
checkFlag(Flag.Type.IS_NULLABLE) to "Nullable flag mismatch",
checkFlag(Flag.Type.IS_SUSPEND) to "Suspend flag mismatch"
)(flags1, flags2)
private fun compareConstructorFlags(flags1: Flags, flags2: Flags): MetadataCompareResult = serialComparator(
checkFlag(Flag.Constructor.IS_SECONDARY) to "IS_SECONDARY mismatch"
)(flags1, flags2)
private fun compare(constructor1: KmConstructor, constructor2: KmConstructor): MetadataCompareResult = serialComparator(
::compareVisibilityFlags,
::compareConstructorFlags
)(constructor1.flags, constructor2.flags)
private fun compare(typeArgument1: KmTypeProjection, typeArgument2: KmTypeProjection): MetadataCompareResult = serialComparator(
compare(KmTypeProjection::type, compareNullable(::compareTypes))
)(typeArgument1, typeArgument2)
private fun compare(typeParameter1: KmTypeParameter, typeParameter2: KmTypeParameter): MetadataCompareResult =
when {
typeParameter1.variance != typeParameter2.variance -> Fail("Different variance")
typeParameter1.name != typeParameter2.name -> Fail("${typeParameter1.name}, ${typeParameter2.name}")
else -> Ok
}
private fun compareTypes(type1: KmType, type2: KmType): MetadataCompareResult = serialComparator(
compare(KmType::classifier, ::compare) to "Classifiers mismatch",
compare(KmType::arguments, compareLists(::compare)) to "Type arguments mismatch",
compare(KmType::flags, ::compareTypeFlags) to "Type flags mismatch for",
compare(KmType::abbreviatedType, compareNullable(::compareTypes)) to "Abbreviated types mismatch"
)(type1, type2)
private fun compare(class1: KmClassifier, class2: KmClassifier): MetadataCompareResult = when {
class1 is KmClassifier.TypeAlias && class2 is KmClassifier.TypeAlias -> {
if (class1.name == class2.name) Ok else Fail("Different type aliases: ${class1.name}, ${class2.name}")
}
class1 is KmClassifier.Class && class2 is KmClassifier.Class -> {
when (class1.name) {
class2.name -> Ok
else -> Fail("Different classes: ${class1.name}, ${class2.name}")
}
}
class1 is KmClassifier.TypeParameter && class2 is KmClassifier.TypeParameter -> {
// TODO: How to correctly compare type ids?
Ok
}
else -> Fail("class1 is $class1 and class2 is $class2")
}
private fun <T, R> compare(
property: T.() -> R,
comparator: (R, R) -> MetadataCompareResult
): (T, T) -> MetadataCompareResult = { o1, o2 ->
if (configuration.shouldCheck(property)) comparator(o1.property(), o2.property()) else Ok
}
}
@@ -0,0 +1,60 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.klib.metadata
import kotlinx.metadata.*
private fun render(element: Any?): String = when (element) {
is KmTypeAlias -> "typealias ${element.name}"
is KmFunction -> "function ${element.name}"
is KmProperty -> "property ${element.name}"
is KmClass -> "class ${element.name}"
is KmType -> "`type ${element.classifier}`"
else -> element.toString()
}
internal fun <T> serialComparator(
vararg comparators: Pair<(T, T) -> MetadataCompareResult, String>?
): (T, T) -> MetadataCompareResult = { o1, o2 ->
comparators.filterNotNull().map { (comparator, message) ->
comparator(o1, o2).let { result ->
if (result is Fail) Fail(message, result) else result
}
}.wrap()
}
internal fun <T> serialComparator(
vararg comparators: (T, T) -> MetadataCompareResult
): (T, T) -> MetadataCompareResult = { o1, o2 ->
comparators
.map { comparator -> comparator(o1, o2) }
.wrap()
}
internal fun Collection<MetadataCompareResult>.wrap(): MetadataCompareResult =
filterIsInstance<Fail>().let { fails ->
when (fails.size) {
0 -> Ok
else -> Fail(fails)
}
}
internal fun <T> compareNullable(
comparator: (T, T) -> MetadataCompareResult
): (T?, T?) -> MetadataCompareResult = { a, b ->
when {
a != null && b != null -> comparator(a, b)
a == null && b == null -> Ok
else -> Fail("${render(a)} ${render(b)}")
}
}
internal fun <T> compareLists(elementComparator: (T, T) -> MetadataCompareResult, sortBy: T.() -> String? = { null }) =
{ list1: List<T>, list2: List<T> -> compareLists(list1.sortedBy(sortBy), list2.sortedBy(sortBy), elementComparator) }
private fun <T> compareLists(l1: List<T>, l2: List<T>, comparator: (T, T) -> MetadataCompareResult) = when {
l1.size != l2.size -> Fail("${l1.size} != ${l2.size}")
else -> l1.zip(l2).map { comparator(it.first, it.second) }.wrap()
}
@@ -0,0 +1,54 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.klib.metadata
import kotlinx.metadata.*
import kotlinx.metadata.klib.KlibModuleFragmentReadStrategy
import kotlinx.metadata.klib.fqName
/**
* Output klib declarations in predictable sorted order.
*/
internal class SortedMergeStrategy : KlibModuleFragmentReadStrategy {
override fun processModuleParts(parts: List<KmModuleFragment>): List<KmModuleFragment> =
parts.fold(KmModuleFragment(), ::joinFragments).let(::listOf)
}
/**
* We need a stable order for overloaded functions.
*/
internal fun KmFunction.mangle(): String {
val typeParameters = typeParameters.joinToString(prefix = "<", postfix = ">", transform = KmTypeParameter::name)
val valueParameters = valueParameters.joinToString(prefix = "(", postfix = ")", transform = KmValueParameter::name)
val receiver = receiverParameterType?.classifier
return "$receiver.${name}.$typeParameters.$valueParameters"
}
internal fun KmProperty.mangle(): String {
val receiver = receiverParameterType?.classifier
return "$receiver.$name"
}
private fun joinAndSortPackages(pkg1: KmPackage, pkg2: KmPackage) = KmPackage().apply {
functions += (pkg1.functions + pkg2.functions).sortedBy(KmFunction::mangle)
properties += (pkg1.properties + pkg2.properties).sortedBy(KmProperty::name)
typeAliases += (pkg1.typeAliases + pkg2.typeAliases).sortedBy(KmTypeAlias::name)
}
/**
* Merges two fragments of a single module into one.
*/
internal fun joinFragments(fragment1: KmModuleFragment, fragment2: KmModuleFragment) = KmModuleFragment().apply {
assert(fragment1.fqName == fragment2.fqName)
pkg = when {
fragment1.pkg != null && fragment2.pkg != null -> joinAndSortPackages(fragment1.pkg!!, fragment2.pkg!!)
fragment1.pkg != null -> joinAndSortPackages(fragment1.pkg!!, KmPackage())
fragment2.pkg != null -> joinAndSortPackages(KmPackage(), fragment2.pkg!!)
else -> null
}
fqName = fragment1.fqName
classes += fragment1.classes.sortedBy(KmClass::name) + fragment2.classes.sortedBy(KmClass::name)
}
@@ -0,0 +1,25 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.klib.metadata
import kotlinx.metadata.klib.KlibModuleMetadata
import org.jetbrains.kotlin.library.MetadataLibrary
/**
* Provides access to metadata using default compiler's routine.
*/
internal class TrivialLibraryProvider(
private val library: MetadataLibrary
) : KlibModuleMetadata.MetadataLibraryProvider {
override val moduleHeaderData: ByteArray
get() = library.moduleHeaderData
override fun packageMetadata(fqName: String, partName: String): ByteArray =
library.packageMetadata(fqName, partName)
override fun packageMetadataParts(fqName: String): Set<String> =
library.packageMetadataParts(fqName)
}
@@ -0,0 +1,202 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.klib.metadata
import kotlinx.metadata.*
import kotlinx.metadata.klib.KlibModuleMetadata
import org.jetbrains.kotlin.konan.file.File
import org.jetbrains.kotlin.library.CompilerSingleFileKlibResolveAllowingIrProvidersStrategy
import org.jetbrains.kotlin.library.resolveSingleFileKlib
private inline fun <reified T : Any> compareElements(
comparisonConfig: ComparisonConfig,
elements: Map<String, Pair<T, T>>,
crossinline comparator: (T, T) -> MetadataCompareResult
): MetadataCompareResult = elements
.entries
.asSequence()
.filter { comparisonConfig.shouldCheckDeclaration(it.key) }
.map { comparator(it.value.first, it.value.second).messageIfFail("${it.key} mismatch") }
.toList()
.wrap()
private class JoinedFragments(
val classes: JoinResult<KmClass>,
val functions: JoinResult<KmFunction>,
val properties: JoinResult<KmProperty>,
val typeAliases: JoinResult<KmTypeAlias>,
)
private fun processMissing(comparisonConfig: ComparisonConfig, joinResult: JoinResult<*>): MetadataCompareResult {
val missingInFirst = joinResult.missingInFirst
.filter(comparisonConfig::shouldCheckDeclaration)
.map { Fail("$it missing in first fragment") }
val missingInSecond = joinResult.missingInSecond
.filter(comparisonConfig::shouldCheckDeclaration)
.map { Fail("$it missing in second fragment") }
return (missingInFirst + missingInSecond).let {
if (it.isEmpty()) Ok else Fail(it)
}
}
private fun MetadataCompareResult.messageIfFail(message: String): MetadataCompareResult =
if (this is Fail) Fail(message, this) else this
private fun processMissing(
comparisonConfig: ComparisonConfig,
joinedFragments: JoinedFragments
): MetadataCompareResult = listOf(
processMissing(comparisonConfig, joinedFragments.classes)
.messageIfFail("Missing classes"),
processMissing(comparisonConfig, joinedFragments.functions)
.messageIfFail("Missing functions"),
processMissing(comparisonConfig, joinedFragments.typeAliases)
.messageIfFail("Missing type aliases"),
processMissing(comparisonConfig, joinedFragments.properties)
.messageIfFail("Missing properties"),
).wrap()
private data class JoinResult<T>(
val joined: Map<String, Pair<T, T>>,
val missingInFirst: List<String>,
val missingInSecond: List<String>
)
private fun <T> buildJoined(e1: List<T>, e2: List<T>, key: T.() -> String): JoinResult<T> {
val m1 = e1.associateBy { it.key() }
val m2 = e2.associateBy { it.key() }
val joinedKeys = e1.map(key).filter { it in m2 }.toSet()
val joined = m1
.filterKeys(joinedKeys::contains)
.mapValues { (key, value) -> value to m2.getValue(key) }
return JoinResult(
joined,
(m1 - joinedKeys).keys.toList(),
(m2 - joinedKeys).keys.toList()
)
}
/**
* Wrapper around direct access to [fragment] that allows
* to uniformly process all its components.
*/
private fun <T> processFragment(
fragment: KmModuleFragment,
action: (List<KmClass>, List<KmFunction>, List<KmProperty>, List<KmTypeAlias>) -> T
): T {
val classes = fragment.classes
val pkg = fragment.pkg
return when {
pkg != null -> action(classes, pkg.functions, pkg.properties, pkg.typeAliases)
else -> action(classes, emptyList(), emptyList(), emptyList())
}
}
private fun compareMetadata(
comparisonConfig: ComparisonConfig,
metadataModuleA: KlibModuleMetadata,
metadataModuleB: KlibModuleMetadata
): MetadataCompareResult {
val fragmentA = metadataModuleA.fragments.fold(KmModuleFragment(), ::joinFragments)
val fragmentB = metadataModuleB.fragments.fold(KmModuleFragment(), ::joinFragments)
val joinedFragments = processFragment(fragmentA) { classesA, functionsA, propertiesA, typeAliasesA ->
processFragment(fragmentB) { classesB, functionsB, propertiesB, typeAliasesB ->
JoinedFragments(
buildJoined(classesA, classesB, KmClass::name),
buildJoined(functionsA, functionsB, KmFunction::name),
buildJoined(propertiesA, propertiesB, KmProperty::name),
buildJoined(typeAliasesA, typeAliasesB, KmTypeAlias::name)
)
}
}
val comparator = KmComparator(comparisonConfig)
return joinedFragments.run {
listOf(
compareElements(comparisonConfig, classes.joined, comparator::compare),
compareElements(comparisonConfig, functions.joined, comparator::compare),
compareElements(comparisonConfig, properties.joined, comparator::compare),
compareElements(comparisonConfig, typeAliases.joined, comparator::compare),
processMissing(comparisonConfig, this)
)
}.wrap()
}
sealed class MetadataCompareResult {
class Fail(
val children: Collection<Fail>, val message: String? = null
) : MetadataCompareResult() {
constructor(message: String, child: Fail? = null)
: this(listOfNotNull(child), message)
}
object Ok : MetadataCompareResult()
}
// A neat way to have short names internally without polluting client's namespace.
internal typealias Fail = MetadataCompareResult.Fail
internal typealias Ok = MetadataCompareResult.Ok
fun expandFail(fail: Fail, output: (String) -> Unit, padding: String = "") {
fail.message?.let { output("$padding$it") }
fail.children.forEach {
expandFail(it, output, "$padding ")
}
}
/**
* Configure what should be tested and what shouldn't.
*
* TODO: Add a way to conditionally disable property comparison.
* E.g. "If class name is Companion do not compare flags".
*/
interface ComparisonConfig {
/**
* Should the declaration be compared at all.
*/
fun shouldCheckDeclaration(element: String): Boolean
/**
* Should we check property of declaration.
*/
fun <T, R> shouldCheck(property: T.() -> R): Boolean
}
/**
* Configuration for comparing `metadata` and `sourcecode` cinterop modes.
*/
class CInteropComparisonConfig : ComparisonConfig {
override fun <T, R> shouldCheck(property: T.() -> R): Boolean = when (property) {
// Kotlin compiler may incorrectly omit abbreviatedType in some cases.
KmType::abbreviatedType -> false
else -> true
}
override fun shouldCheckDeclaration(element: String): Boolean = when {
// kniBridge is generated only in sourcecode mode.
element.startsWith("kniBridge") -> false
else -> true
}
}
/**
* Structurally compares metadata of given libraries.
*/
fun compareKlibMetadata(
comparisonConfig: ComparisonConfig,
pathToFirstLibrary: String,
pathToSecondLibrary: String
): MetadataCompareResult {
val resolveStrategy = CompilerSingleFileKlibResolveAllowingIrProvidersStrategy(
knownIrProviders = listOf("kotlin.native.cinterop")
)
val klib1 = resolveSingleFileKlib(File(pathToFirstLibrary), strategy = resolveStrategy)
val klib2 = resolveSingleFileKlib(File(pathToSecondLibrary), strategy = resolveStrategy)
val metadata1 = KlibModuleMetadata.read(TrivialLibraryProvider(klib1), SortedMergeStrategy())
val metadata2 = KlibModuleMetadata.read(TrivialLibraryProvider(klib2), SortedMergeStrategy())
return compareMetadata(comparisonConfig, metadata1, metadata2)
}
@@ -0,0 +1,170 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.testing.native
import org.gradle.api.DefaultTask
import org.gradle.api.provider.Property
import org.gradle.api.provider.Provider
import org.gradle.api.tasks.TaskAction
import org.gradle.api.tasks.options.Option
import org.gradle.process.ExecResult
import org.gradle.process.ExecSpec
import org.jetbrains.kotlin.isNotEmpty
import java.io.File
import java.net.URL
import java.util.*
import javax.inject.Inject
/**
* Clones the given revision of the given Git repository to the given directory.
*/
@Suppress("UnstableApiUsage")
open class GitDownloadTask @Inject constructor(
val repositoryProvider: Provider<URL>,
val revisionProvider: Provider<String>,
val outputDirectoryProvider: Provider<File>
) : DefaultTask() {
private val repository: URL
get() = repositoryProvider.get()
private val revision: String
get() = revisionProvider.get()
private val outputDirectory: File
get() = outputDirectoryProvider.get()
@Option(option = "refresh",
description = "Fetch and checkout the revision even if the output directory already contains it. " +
"All changes in the output directory will be overwritten")
val refresh: Property<Boolean> = project.objects.property(Boolean::class.java).apply {
set(false)
}
private val upToDateChecker = UpToDateChecker()
init {
outputs.upToDateWhen { upToDateChecker.isUpToDate() }
}
private fun git(
vararg args: String,
ignoreExitValue: Boolean = false,
execConfiguration: ExecSpec.() -> Unit = {}
): ExecResult =
project.exec {
it.executable = "git"
it.args(*args)
it.isIgnoreExitValue = ignoreExitValue
it.execConfiguration()
}
private fun tryCloneBranch(): Boolean {
val execResult = git(
"clone", repository.toString(),
outputDirectory.absolutePath,
"--depth", "1",
"--branch", revision,
ignoreExitValue = true
)
return execResult.exitValue == 0
}
private fun fetchByHash() {
git("init", outputDirectory.absolutePath)
git("fetch", repository.toString(), "--depth", "1", revision) {
workingDir(outputDirectory)
}
git("reset", "--hard", revision) {
workingDir(outputDirectory)
}
}
@TaskAction
fun clone() {
// Gradle ignores outputs.upToDateWhen { ... } and reruns a task if the classpath of the task was changed
// So we have to perform the up-to-date check manually one more time in the task action.
if (upToDateChecker.isUpToDate()) {
logger.info("Skip cloning to avoid rewriting possible debug changes in ${outputDirectory.absolutePath}.")
return
}
project.delete {
it.delete(outputDirectory)
}
if (!tryCloneBranch()) {
logger.info("Cannot use the revision '$revision' to clone the repository. Trying to use init && fetch instead.")
fetchByHash()
}
// Store info about used revision for the manual up-to-date check.
upToDateChecker.storeRevisionInfo()
}
/**
* This class performs manual UP-TO-DATE checking.
*
* We want to be able to edit downloaded sources for debug purposes. Thus this task should not rewrite changes in
* the output directory.
*
* Gradle does allow us to provide a custom logic to determine if task outputs are UP-TO-DATE or not (see
* Task.outputs.upToDateWhen). But Gradle still reruns a task if its classpath was changed. This may lead
* to rewriting our manual debug changes in downloaded sources if there are some changes in the
* `build-tools` project.
*
* So we have to manually check up-to-dateness in upToDateWhen and as a first step of the task action.
*/
private inner class UpToDateChecker() {
private val revisionInfoFile: File
get() = outputDirectory.resolve(".revision")
/**
* The download task should be executed in the following cases:
*
* - The output directory doesn't exist or is empty;
* - Repository or revision was changed since the last execution;
* - A user forced rerunning this tasks manually (see [GitDownloadTask.refresh]).
*
* In all other cases we consider the task UP-TO-DATE.
*/
fun isUpToDate(): Boolean {
return !refresh.get() &&
outputDirectory.let { it.exists() && project.fileTree(it).isNotEmpty } &&
noRevisionChanges()
}
private fun noRevisionChanges(): Boolean =
revisionInfoFile.exists() && loadRevisionInfo() == RevisionInfo(repository, revision)
fun storeRevisionInfo() {
val properties = Properties()
properties["repository"] = repository.toString()
properties["revision"] = revision
revisionInfoFile.bufferedWriter().use {
properties.store(it, null)
}
}
private fun loadRevisionInfo(): RevisionInfo? {
return try {
val properties = Properties()
revisionInfoFile.bufferedReader().use {
properties.load(it)
}
RevisionInfo(properties.getProperty("repository"), properties.getProperty("revision"))
} catch (_ : Exception) {
null
}
}
}
private data class RevisionInfo(val repository: String, val revision: String) {
constructor(repository: URL, revision: String): this(repository.toString(), revision)
}
}
@@ -0,0 +1,261 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.testing.native
import groovy.lang.Closure
import java.io.File
import javax.inject.Inject
import org.gradle.api.*
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.tasks.*
import org.jetbrains.kotlin.ExecClang
import org.jetbrains.kotlin.bitcode.CompileToBitcode
import org.jetbrains.kotlin.konan.target.*
open class CompileNativeTest @Inject constructor(
@InputFile val inputFile: File,
@Input val target: String
) : DefaultTask() {
@OutputFile
var outputFile = project.buildDir.resolve("bin/test/$target/${inputFile.nameWithoutExtension}.o")
@Input
val clangArgs = mutableListOf<String>()
@TaskAction
fun compile() {
val plugin = project.convention.getPlugin(ExecClang::class.java)
plugin.execBareClang {
it.executable = "clang++"
it.args = clangArgs + listOf(inputFile.absolutePath, "-o", outputFile.absolutePath)
}
}
}
open class LlvmLinkNativeTest @Inject constructor(
val baseName: String,
@Input val target: String,
@InputFile val mainFile: File
) : DefaultTask() {
@SkipWhenEmpty
@InputFiles
var inputFiles: ConfigurableFileCollection = project.files()
@OutputFile
var outputFile: File = project.buildDir.resolve("bitcode/test/$target/$baseName.bc")
@TaskAction
fun llvmLink() {
val llvmDir = project.property("llvmDir")
val tmpOutput = File.createTempFile("runtimeTests", ".bc").apply {
deleteOnExit()
}
// The runtime provides our implementations for some standard functions (see StdCppStubs.cpp).
// We need to internalize these symbols to avoid clashes with symbols provided by the C++ stdlib.
// But llvm-link -internalize is kinda broken: it links modules one by one and can't see usages
// of a symbol in subsequent modules. So it will mangle such symbols causing "unresolved symbol"
// errors at the link stage. So we have to run llvm-link twice: the first one links all modules
// except the one containing the entry point to a single *.bc without internalization. The second
// run internalizes this big module and links it with a module containing the entry point.
project.exec {
it.executable = "$llvmDir/bin/llvm-link"
it.args = listOf("-o", tmpOutput.absolutePath) + inputFiles.map { it.absolutePath }
}
project.exec {
it.executable = "$llvmDir/bin/llvm-link"
it.args = listOf(
"-o", outputFile.absolutePath,
mainFile.absolutePath,
tmpOutput.absolutePath,
"-internalize"
)
}
}
}
open class LinkNativeTest @Inject constructor(
@InputFiles val inputFiles: List<File>,
@OutputFile val outputFile: File,
@Internal val target: String,
@Internal val linkerArgs: List<String>,
private val platformManager: PlatformManager,
private val mimallocEnabled: Boolean
) : DefaultTask () {
companion object {
fun create(
project: Project,
platformManager: PlatformManager,
taskName: String,
inputFiles: List<File>,
target: String,
outputFile: File,
linkerArgs: List<String>,
mimallocEnabled: Boolean
): LinkNativeTest = project.tasks.create(
taskName,
LinkNativeTest::class.java,
inputFiles,
outputFile,
target,
linkerArgs,
platformManager,
mimallocEnabled)
fun create(
project: Project,
platformManager: PlatformManager,
taskName: String,
inputFiles: List<File>,
target: String,
executableName: String,
mimallocEnabled: Boolean,
linkerArgs: List<String> = listOf()
): LinkNativeTest = create(
project,
platformManager,
taskName,
inputFiles,
target,
project.buildDir.resolve("bin/test/$target/$executableName"),
linkerArgs, mimallocEnabled)
}
@get:Input
val commands: List<List<String>>
get() {
// Getting link commands requires presence of a target toolchain.
// Thus we cannot get them at the configuration stage because the toolchain may be not downloaded yet.
val linker = platformManager.platform(platformManager.targetByName(target)).linker
return linker.finalLinkCommands(
inputFiles.map { it.absolutePath },
outputFile.absolutePath,
listOf(),
linkerArgs,
optimize = false,
debug = false,
kind = LinkerOutputKind.EXECUTABLE,
outputDsymBundle = "",
needsProfileLibrary = false,
mimallocEnabled = mimallocEnabled
).map { it.argsWithExecutable }
}
@TaskAction
fun link() {
for (command in commands) {
project.exec {
it.commandLine(command)
}
}
}
}
fun createTestTask(
project: Project,
testName: String,
testTaskName: String,
testedTaskNames: List<String>,
configureCompileToBitcode: CompileToBitcode.() -> Unit = {},
): Task {
val platformManager = project.rootProject.findProperty("platformManager") as PlatformManager
val googleTestExtension = project.extensions.getByName(RuntimeTestingPlugin.GOOGLE_TEST_EXTENSION_NAME) as GoogleTestExtension
val testedTasks = testedTaskNames.map {
project.tasks.getByName(it) as CompileToBitcode
}
val target = testedTasks.map {
it.target
}.distinct().single()
val konanTarget = platformManager.targetByName(target)
val compileToBitcodeTasks = testedTasks.mapNotNull {
val name = "${it.name}TestBitcode"
val task = project.tasks.findByName(name) as? CompileToBitcode ?:
project.tasks.create(name,
CompileToBitcode::class.java,
it.srcRoot,
"${it.folderName}Tests",
target, "test"
).apply {
excludeFiles = emptyList()
includeFiles = listOf("**/*Test.cpp", "**/*Test.mm")
dependsOn(it)
compilerArgs.addAll(it.compilerArgs)
headersDirs += googleTestExtension.headersDirs
this.configureCompileToBitcode()
}
if (task.inputFiles.count() == 0)
null
else
task
}
val testFrameworkTasks = listOf(
project.tasks.getByName("${target}Googletest") as CompileToBitcode,
project.tasks.getByName("${target}Googlemock") as CompileToBitcode
)
val testSupportTask = project.tasks.getByName("${target}TestSupport") as CompileToBitcode
// TODO: It may make sense to merge llvm-link, compile and link to a single task.
val llvmLinkTask = project.tasks.create(
"${testTaskName}LlvmLink",
LlvmLinkNativeTest::class.java,
testTaskName, target, testSupportTask.outFile
).apply {
val tasksToLink = (compileToBitcodeTasks + testedTasks + testFrameworkTasks)
inputFiles = project.files(tasksToLink.map { it.outFile })
dependsOn(testSupportTask)
dependsOn(tasksToLink)
}
val clangFlags = platformManager.platform(konanTarget).configurables as ClangFlags
val compileTask = project.tasks.create(
"${testTaskName}Compile",
CompileNativeTest::class.java,
llvmLinkTask.outputFile,
target
).apply {
dependsOn(llvmLinkTask)
clangArgs.addAll(clangFlags.clangFlags)
clangArgs.addAll(clangFlags.clangNooptFlags)
}
val mimallocEnabled = testedTaskNames.any { it.contains("mimalloc", ignoreCase = true) }
val linkTask = LinkNativeTest.create(
project,
platformManager,
"${testTaskName}Link",
listOf(compileTask.outputFile),
target,
testTaskName,
mimallocEnabled
).apply {
dependsOn(compileTask)
}
return project.tasks.create(testTaskName, Exec::class.java).apply {
dependsOn(linkTask)
workingDir = project.buildDir.resolve("testReports/$testTaskName")
val xmlReport = workingDir.resolve("report.xml")
executable(linkTask.outputFile)
args("--gtest_output=xml:${xmlReport.absoluteFile}")
doFirst {
workingDir.mkdirs()
}
doLast {
// TODO: Better to use proper XML parsing.
var contents = xmlReport.readText()
contents = contents.replace("<testsuite name=\"", "<testsuite name=\"${testName}.")
contents = contents.replace("classname=\"", "classname=\"${testName}.")
val rewrittenReport = workingDir.resolve("report-with-prefixes.xml")
rewrittenReport.writeText(contents)
}
}
}
@@ -0,0 +1,159 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the LICENSE file.
*/
package org.jetbrains.kotlin.testing.native
import org.gradle.api.InvalidUserDataException
import org.gradle.api.Plugin
import org.gradle.api.Project
import org.gradle.api.file.FileCollection
import org.gradle.api.provider.Provider
import org.gradle.api.tasks.TaskProvider
import org.jetbrains.kotlin.bitcode.CompileToBitcodeExtension
import org.jetbrains.kotlin.bitcode.CompileToBitcodePlugin
import org.jetbrains.kotlin.resolve
import java.io.File
import java.net.URL
import javax.inject.Inject
@Suppress("UnstableApiUsage")
open class RuntimeTestingPlugin : Plugin<Project> {
override fun apply(target: Project): Unit = with(target) {
val extension = extensions.create(GOOGLE_TEST_EXTENSION_NAME, GoogleTestExtension::class.java, target)
val downloadTask = registerDownloadTask(extension)
val googleTestRoot = project.provider { extension.sourceDirectory }
createBitcodeTasks(googleTestRoot, listOf(downloadTask))
}
private fun Project.registerDownloadTask(extension: GoogleTestExtension): TaskProvider<GitDownloadTask> {
val task = tasks.register(
"downloadGoogleTest",
GitDownloadTask::class.java,
provider { URL(extension.repository) },
provider { extension.revision },
provider { extension.fetchDirectory }
)
task.configure {
it.refresh.set(provider { extension.refresh })
it.onlyIf { extension.localSourceRoot == null }
it.description = "Retrieves GoogleTest from the given repository"
it.group = "Google Test"
}
return task
}
private fun Project.createBitcodeTasks(
googleTestRoot: Provider<File>,
dependencies: Iterable<TaskProvider<*>>
) {
pluginManager.withPlugin("compile-to-bitcode") {
val bitcodeExtension =
project.extensions.getByName(CompileToBitcodePlugin.EXTENSION_NAME) as CompileToBitcodeExtension
bitcodeExtension.create("googletest", outputGroup = "test") {
srcDirs = project.files(
googleTestRoot.resolve("googletest/src")
)
headersDirs = project.files(
googleTestRoot.resolve("googletest/include"),
googleTestRoot.resolve("googletest")
)
includeFiles = listOf("*.cc")
excludeFiles = listOf("gtest-all.cc", "gtest_main.cc")
// Original GTest sources contain an unused variable on Windows (kAlternatePathSeparatorString).
compilerArgs.add("-Wno-unused")
dependsOn(dependencies)
}
bitcodeExtension.create("googlemock", outputGroup = "test") {
srcDirs = project.files(
googleTestRoot.resolve("googlemock/src")
)
headersDirs = project.files(
googleTestRoot.resolve("googlemock"),
googleTestRoot.resolve("googlemock/include"),
googleTestRoot.resolve("googletest/include")
)
includeFiles = listOf("*.cc")
excludeFiles = listOf("gmock-all.cc", "gmock_main.cc")
dependsOn(dependencies)
}
}
}
companion object {
internal const val GOOGLE_TEST_EXTENSION_NAME = "googletest"
}
}
/**
* A project extension to configure from where we get the GoogleTest framework.
*/
open class GoogleTestExtension @Inject constructor(private val project: Project) {
/**
* A repository to fetch GoogleTest from.
*/
var repository: String = "https://github.com/google/googletest.git"
private var _revision: String? = null
/**
* A particular revision in the [repository] to be fetched. It can be a branch, a tag or a commit hash.
*/
var revision: String
get() = _revision
?: throw InvalidUserDataException(
"No value provided for property '${RuntimeTestingPlugin.GOOGLE_TEST_EXTENSION_NAME}.revision'. " +
"Please specify it in the buildscript."
)
set(value) { _revision = value }
/**
* Fetch the [revision] even if the [fetchDirectory] already contains it. Overwrite all changes manually made in the output directory.
*/
var refresh: Boolean = false
/**
* A directory to fetch the [revision] to.
*/
var fetchDirectory: File = project.file("googletest")
internal var localSourceRoot: File? = null
/**
* Use a local [directory] with GoogleTest instead of the fetched one. If set, the download task will not be executed.
*/
fun useLocalSources(directory: File) {
localSourceRoot = directory
}
/**
* Use a local [directory] with GoogleTest instead of the fetched one. If set, the download task will not be executed.
*/
fun useLocalSources(directory: String) {
localSourceRoot = project.file(directory)
}
/**
* A getter for directory that contains the GTest sources.
* Returns a local source directory if it's specified (see [useLocalSources]) or [fetchDirectory] otherwise.
*/
val sourceDirectory: File
get() = localSourceRoot ?: fetchDirectory
/**
* A file collection with header directories for GoogleTest and GoogleMock.
* Useful to configure compilation against GTest.
*/
val headersDirs: FileCollection = project.files(
project.provider { sourceDirectory.resolve("googletest/include") },
project.provider { sourceDirectory.resolve("googlemock/include") }
)
}
@@ -0,0 +1,121 @@
package org.jetbrains.kotlin.utils
import java.util.*
// Copied from Kotlin org.jetbrains.kotlin.utils.DFS class
@Suppress("MemberVisibilityCanBePrivate", "MemberVisibilityCanBePrivate", "unused")
object DFS {
fun <N, R> dfs(nodes: Collection<N>, neighbors: Neighbors<N>, visited: Visited<N>, handler: NodeHandler<N, R>): R {
for (node in nodes) {
doDfs(node, neighbors, visited, handler)
}
return handler.result()
}
fun <N, R> dfs(
nodes: Collection<N>,
neighbors: Neighbors<N>,
handler: NodeHandler<N, R>
): R {
return dfs(nodes, neighbors, VisitedWithSet(), handler)
}
fun <N> ifAny(
nodes: Collection<N>,
neighbors: Neighbors<N>,
predicate: Function1<N, Boolean>
): Boolean {
val result = BooleanArray(1)
return dfs(nodes, neighbors, object : AbstractNodeHandler<N, Boolean?>() {
override fun beforeChildren(current: N): Boolean {
if (predicate.invoke(current)) {
result[0] = true
}
return !result[0]
}
override fun result(): Boolean {
return result[0]
}
})!!
}
fun <N, R> dfsFromNode(node: N, neighbors: Neighbors<N>, visited: Visited<N>, handler: NodeHandler<N, R>): R {
doDfs(node, neighbors, visited, handler)
return handler.result()
}
fun <N> dfsFromNode(
node: N,
neighbors: Neighbors<N>,
visited: Visited<N>
) {
dfsFromNode(node, neighbors, visited, object : AbstractNodeHandler<N, Void?>() {
override fun result(): Void? {
return null
}
})
}
fun <N> topologicalOrder(nodes: Iterable<N>, neighbors: Neighbors<N>, visited: Visited<N>): List<N> {
val handler = TopologicalOrder<N>()
for (node in nodes) {
doDfs(node, neighbors, visited, handler)
}
return handler.result()
}
fun <N> topologicalOrder(nodes: Iterable<N>, neighbors: Neighbors<N>): List<N> {
return topologicalOrder(nodes, neighbors, VisitedWithSet())
}
fun <N> doDfs(current: N, neighbors: Neighbors<N>, visited: Visited<N>, handler: NodeHandler<N, *>) {
if (!visited.checkAndMarkVisited(current)) return
if (!handler.beforeChildren(current)) return
for (neighbor in neighbors.getNeighbors(current)) {
doDfs(neighbor, neighbors, visited, handler)
}
handler.afterChildren(current)
}
interface NodeHandler<N, R> {
fun beforeChildren(current: N): Boolean
fun afterChildren(current: N)
fun result(): R
}
interface Neighbors<N> {
fun getNeighbors(current: N): Iterable<N>
}
interface Visited<N> {
fun checkAndMarkVisited(current: N): Boolean
}
abstract class AbstractNodeHandler<N, R> : NodeHandler<N, R> {
override fun beforeChildren(current: N): Boolean {
return true
}
override fun afterChildren(current: N) {}
}
class VisitedWithSet<N> @JvmOverloads constructor(private val visited: MutableSet<N> = HashSet()) : Visited<N> {
override fun checkAndMarkVisited(current: N): Boolean {
return visited.add(current)
}
}
abstract class CollectingNodeHandler<N, R, C : Iterable<R>> protected constructor(protected val result: C) : AbstractNodeHandler<N, C>() {
override fun result(): C {
return result
}
}
abstract class NodeHandlerWithListResult<N, R> protected constructor() : CollectingNodeHandler<N, R, LinkedList<R>>(LinkedList<R>())
class TopologicalOrder<N> : NodeHandlerWithListResult<N, N>() {
override fun afterChildren(current: N) {
result.addFirst(current)
}
}
}