[JS IR] Build stdlib using compiler from repo

Previously JS IR versions of stdlib and kotlin-test were build
by default using compiler previously built on a buildserver.

It had some issues:
 - This required us to advance bootstrap every time we made any
   incompatible IR changes. This happens often since IR ABI is
   not quite stable yet.

 - We never tested the exact combination of compiler and stdlib we publish

   We tested:
    - new compiler with new stdlib build by new compiler (in box tests)
    - old compiler with new stdlib build by old compiler (in stdlib tests)

   We published:
    - new compiler with new stdlib build by old compiler

After this change JS IR compiler tests, builds and publishes
single configuration:

    new compiler with new stdlib build by new compiler

JS IR stdlib and kotlin-test are now built using JavaExec of CLI instead
of Gradle plugin to avoid troubles of loading a freshly built plugin.

This also allows to have a granular dependencies: we don't rebuild klib
if we changed a lowering in a compiler backend, but we do rebuild it if
we changed IR serialization algorithm.
This commit is contained in:
Svyatoslav Kuzmich
2019-11-18 20:17:21 +03:00
parent 70111f8db1
commit 31c84e9ac4
20 changed files with 439 additions and 420 deletions
-198
View File
@@ -1,198 +0,0 @@
plugins {
id "com.github.node-gradle.node"
}
description = 'Kotlin Standard Library for JS IR'
apply plugin: 'kotlin-platform-js'
apply plugin: 'idea'
def originalBuiltinsDir = "${rootDir}/core/builtins"
def builtinsSrcDir = "${buildDir}/builtin-sources"
def nativeBuiltinsSrcDir = "${buildDir}/native-builtin-sources"
// Sync native built-ins to separate directory because we don't
// need them all -- some of them are implemented as proper code in 'js-ir/builtins'.
// Also, they don't compile on its own without error suppressing.
task prepareNativeBuiltinsSources(type: Sync) {
from("${originalBuiltinsDir}/native/kotlin") {
include "Annotation.kt"
include "Any.kt"
include "Array.kt"
include "Boolean.kt"
include "CharSequence.kt"
include "Comparable.kt"
include "Coroutines.kt"
include "Iterator.kt"
include "Nothing.kt"
include "Number.kt"
include "String.kt"
}
into nativeBuiltinsSrcDir
doLast {
(new File(nativeBuiltinsSrcDir)).list().each { path ->
def file = new File("${nativeBuiltinsSrcDir}/${path}")
def sourceCode = """\
@file:Suppress(
"NON_ABSTRACT_FUNCTION_WITH_NO_BODY",
"MUST_BE_INITIALIZED_OR_BE_ABSTRACT",
"EXTERNAL_TYPE_EXTENDS_NON_EXTERNAL_TYPE",
"PRIMARY_CONSTRUCTOR_DELEGATION_CALL_EXPECTED",
"WRONG_MODIFIER_TARGET"
)
""".stripIndent() + file.getText()
file.setText(sourceCode)
}
}
}
task prepareBuiltinsSources(type: Sync) {
from("${originalBuiltinsDir}/src/kotlin/") {
exclude "ArrayIntrinsics.kt" // Used only for JVM
}
into builtinsSrcDir
}
def jsCommonDir = "${buildDir}/common-js-sources"
task prepareCommonJsSources(type: Sync) {
from("${projectDir}/../js/")
exclude 'src/generated/**' // is used for legacy backend
into jsCommonDir
}
sourceSets {
main {
kotlin {
srcDir "${projectDir}/builtins"
srcDir "${projectDir}/runtime"
srcDir "${projectDir}/src"
srcDir "${jsCommonDir}/src"
srcDir "${jsCommonDir}/runtime"
srcDir builtinsSrcDir
srcDir nativeBuiltinsSrcDir
}
}
test {
kotlin {
srcDir "${jsCommonDir}/test/"
}
}
}
configurations {
commonSources
}
dependencies {
expectedBy project(":kotlin-stdlib-common")
commonSources project(path: ":kotlin-stdlib-common", configuration: "sources")
testCompile project(':kotlin-test:kotlin-test-js')
}
compileKotlin2Js {
dependsOn prepareNativeBuiltinsSources
dependsOn prepareBuiltinsSources
dependsOn prepareCommonJsSources
kotlinOptions {
outputFile = "${buildDir}/classes/main/kotlin.js"
moduleKind = "umd"
main = "noCall"
verbose = true
freeCompilerArgs += [
"-Xir-produce-klib-dir",
"-Xir-only",
"-Xallow-kotlin-package",
"-Xallow-result-return-type",
"-Xuse-experimental=kotlin.Experimental",
"-Xuse-experimental=kotlin.ExperimentalMultiplatform",
"-Xuse-experimental=kotlin.contracts.ExperimentalContracts",
"-Xinline-classes",
]
}
}
def testOutputFile = "${buildDir}/classes/kotlin/test/kotlin-stdlib-js-ir_test.js"
def kotlinTestTestOutputFile = "${project(':kotlin-test:kotlin-test-js').buildDir}/classes/kotlin/test/kotlin-test-js-ir_test.js"
compileTestKotlin2Js {
kotlinOptions {
moduleKind = "umd"
verbose = true
outputFile = testOutputFile
freeCompilerArgs += [
"-Xir-produce-js",
"-verbose",
"-Xuse-experimental=kotlin.Experimental",
"-Xuse-experimental=kotlin.ExperimentalUnsignedTypes",
"-Xuse-experimental=kotlin.ExperimentalStdlibApi"
]
}
}
archivesBaseName = 'kotlin-stdlib-js-ir'
jar {
manifestAttributes(manifest, project, 'Main')
duplicatesStrategy = DuplicatesStrategy.FAIL
}
task sourcesJar(type: Jar, dependsOn: classes) {
classifier = 'sources'
duplicatesStrategy = DuplicatesStrategy.FAIL
from (sourceSets.main.allSource)
}
artifacts {
archives sourcesJar
}
node {
download = true
version = '8.9.4' // The default 6.9.1 has buggy hyperbolic functions implementation
nodeModulesDir = buildDir
}
task install() {
// dummy task to make coreLibsInstall aggregate task not fail
}
task installMocha(type: NpmTask) {
args = ['install', 'mocha']
}
task installTeamcityReporter(type: NpmTask) {
args = ['install', 'mocha-teamcity-reporter']
}
task runMocha(type: NodeTask) {
dependsOn compileTestKotlin2Js
dependsOn installMocha
dependsOn ':kotlin-test:kotlin-test-js:testClasses'
script = file("${buildDir}/node_modules/mocha/bin/mocha")
if (project.hasProperty("teamcity")) {
dependsOn installTeamcityReporter
args = ['--reporter', 'mocha-teamcity-reporter',
'--reporter-options', 'topLevelSuite=stdlib-js-ir']
}
else {
args = ['--reporter', 'min']
}
args += [testOutputFile, kotlinTestTestOutputFile]
execOverrides {
it.ignoreExitValue = rootProject.ignoreTestFailures
it.workingDir = buildDir
}
}
test.dependsOn runMocha
+339
View File
@@ -0,0 +1,339 @@
import com.moowork.gradle.node.npm.NpmTask
import com.moowork.gradle.node.task.NodeTask
plugins {
kotlin("jvm")
id("com.github.node-gradle.node")
}
// A simple CLI for creating JS IR klibs.
// Does not depend on backend lowerings and JS codegen.
val jsIrKlibCli: Configuration by configurations.creating
// Full JS IR compiler CLI
val fullJsIrCli: Configuration by configurations.creating
dependencies {
jsIrKlibCli(project(":compiler:cli-js-klib"))
fullJsIrCli(project(":compiler:cli-js"))
fullJsIrCli(project(":compiler:util"))
fullJsIrCli(project(":compiler:cli-common"))
fullJsIrCli(project(":compiler:cli"))
fullJsIrCli(project(":compiler:frontend"))
fullJsIrCli(project(":compiler:backend-common"))
fullJsIrCli(project(":compiler:backend"))
fullJsIrCli(project(":compiler:ir.backend.common"))
fullJsIrCli(project(":compiler:ir.serialization.js"))
fullJsIrCli(project(":compiler:backend.js"))
fullJsIrCli(project(":js:js.translator"))
fullJsIrCli(project(":js:js.serializer"))
fullJsIrCli(project(":js:js.dce"))
fullJsIrCli(kotlin("reflect"))
fullJsIrCli(intellijCoreDep()) { includeJars("intellij-core") }
fullJsIrCli(intellijDep()) {
includeJars("picocontainer", "trove4j", "guava", "jdom", "asm-all", rootProject = rootProject)
}
}
val unimplementedNativeBuiltIns =
(file("$rootDir/core/builtins/native/kotlin/").list().toSet() - file("$rootDir/libraries/stdlib/js-ir/builtins/").list())
.map { "core/builtins/native/kotlin/$it" }
// Required to compile native builtins with the rest of runtime
val builtInsHeader = """@file:Suppress(
"NON_ABSTRACT_FUNCTION_WITH_NO_BODY",
"MUST_BE_INITIALIZED_OR_BE_ABSTRACT",
"EXTERNAL_TYPE_EXTENDS_NON_EXTERNAL_TYPE",
"PRIMARY_CONSTRUCTOR_DELEGATION_CALL_EXPECTED",
"WRONG_MODIFIER_TARGET"
)
"""
val fullRuntimeSources by task<Sync> {
val sources = listOf(
"core/builtins/src/kotlin/",
"libraries/stdlib/common/src/",
"libraries/stdlib/src/kotlin/",
"libraries/stdlib/unsigned/",
"libraries/stdlib/js/src/",
"libraries/stdlib/js/runtime/",
"libraries/stdlib/js-ir/builtins/",
"libraries/stdlib/js-ir/src/",
"libraries/stdlib/js-ir/runtime/",
// TODO get rid - move to test module
"js/js.translator/testData/_commonFiles/"
) + unimplementedNativeBuiltIns
val excluded = listOf(
// stdlib/js/src/generated is used exclusively for current `js-v1` backend.
"libraries/stdlib/js/src/generated/**",
// JS-specific optimized version of emptyArray() already defined
"core/builtins/src/kotlin/ArrayIntrinsics.kt"
)
sources.forEach { path ->
from("$rootDir/$path") {
into(path.dropLastWhile { it != '/' })
excluded.filter { it.startsWith(path) }.forEach {
exclude(it.substring(path.length))
}
}
}
into("$buildDir/fullRuntime/src")
doLast {
unimplementedNativeBuiltIns.forEach { path ->
val file = File("$buildDir/fullRuntime/src/$path")
val sourceCode = builtInsHeader + file.readText()
file.writeText(sourceCode)
}
}
}
val reducedRuntimeSources by task<Sync> {
dependsOn(fullRuntimeSources)
from(fullRuntimeSources.get().outputs.files.singleFile) {
exclude(
listOf(
"libraries/stdlib/unsigned/**",
"libraries/stdlib/common/src/generated/_Arrays.kt",
"libraries/stdlib/common/src/generated/_Collections.kt",
"libraries/stdlib/common/src/generated/_Comparisons.kt",
"libraries/stdlib/common/src/generated/_Maps.kt",
"libraries/stdlib/common/src/generated/_Sequences.kt",
"libraries/stdlib/common/src/generated/_Sets.kt",
"libraries/stdlib/common/src/generated/_Strings.kt",
"libraries/stdlib/common/src/generated/_UArrays.kt",
"libraries/stdlib/common/src/generated/_URanges.kt",
"libraries/stdlib/common/src/generated/_UCollections.kt",
"libraries/stdlib/common/src/generated/_UComparisons.kt",
"libraries/stdlib/common/src/generated/_USequences.kt",
"libraries/stdlib/common/src/kotlin/SequencesH.kt",
"libraries/stdlib/common/src/kotlin/TextH.kt",
"libraries/stdlib/common/src/kotlin/UMath.kt",
"libraries/stdlib/common/src/kotlin/collections/**",
"libraries/stdlib/common/src/kotlin/ioH.kt",
"libraries/stdlib/js-ir/runtime/collectionsHacks.kt",
"libraries/stdlib/js-ir/src/generated/**",
"libraries/stdlib/js-ir/src/kotlin/text/**",
"libraries/stdlib/js/src/jquery/**",
"libraries/stdlib/js/src/org.w3c/**",
"libraries/stdlib/js/src/kotlin/char.kt",
"libraries/stdlib/js/src/kotlin/collections.kt",
"libraries/stdlib/js/src/kotlin/collections/**",
"libraries/stdlib/js/src/kotlin/time/**",
"libraries/stdlib/js/src/kotlin/console.kt",
"libraries/stdlib/js/src/kotlin/coreDeprecated.kt",
"libraries/stdlib/js/src/kotlin/date.kt",
"libraries/stdlib/js/src/kotlin/debug.kt",
"libraries/stdlib/js/src/kotlin/grouping.kt",
"libraries/stdlib/js/src/kotlin/json.kt",
"libraries/stdlib/js/src/kotlin/promise.kt",
"libraries/stdlib/js/src/kotlin/regexp.kt",
"libraries/stdlib/js/src/kotlin/sequence.kt",
"libraries/stdlib/js/src/kotlin/text/**",
"libraries/stdlib/js/src/kotlin/reflect/KTypeHelpers.kt",
"libraries/stdlib/js/src/kotlin/reflect/KTypeParameterImpl.kt",
"libraries/stdlib/js/src/kotlin/reflect/KTypeImpl.kt",
"libraries/stdlib/src/kotlin/collections/**",
"libraries/stdlib/src/kotlin/experimental/bitwiseOperations.kt",
"libraries/stdlib/src/kotlin/properties/Delegates.kt",
"libraries/stdlib/src/kotlin/random/URandom.kt",
"libraries/stdlib/src/kotlin/text/**",
"libraries/stdlib/src/kotlin/time/**",
"libraries/stdlib/src/kotlin/util/KotlinVersion.kt",
"libraries/stdlib/src/kotlin/util/Tuples.kt",
"libraries/stdlib/js/src/kotlin/dom/**",
"libraries/stdlib/js/src/kotlin/browser/**"
)
)
}
from("$rootDir/libraries/stdlib/js-ir/smallRuntime") {
into("libraries/stdlib/js-ir/runtime/")
}
into("$buildDir/reducedRuntime/src")
}
fun JavaExec.buildKLib(sources: List<String>, dependencies: List<String>, outPath: String, commonSources: List<String>) {
inputs.files(sources)
outputs.dir(file(outPath).parent)
classpath = jsIrKlibCli
main = "org.jetbrains.kotlin.ir.backend.js.GenerateJsIrKlibKt"
workingDir = rootDir
args = sources.toList() + listOf("-o", outPath) + dependencies.flatMap { listOf("-d", it) } + commonSources.flatMap { listOf("-c", it) }
dependsOn(":compiler:cli-js-klib:jar")
passClasspathInJar()
}
val fullRuntimeDir = buildDir.resolve("fullRuntime/klib")
val generateFullRuntimeKLib by eagerTask<NoDebugJavaExec> {
dependsOn(fullRuntimeSources)
buildKLib(sources = listOf(fullRuntimeSources.get().outputs.files.singleFile.path),
dependencies = emptyList(),
outPath = fullRuntimeDir.absolutePath,
commonSources = listOf("common", "src", "unsigned").map { "$buildDir/fullRuntime/src/libraries/stdlib/$it" }
)
}
val packFullRuntimeKLib by tasks.registering(Jar::class) {
dependsOn(generateFullRuntimeKLib)
from(fullRuntimeDir)
destinationDirectory.set(rootProject.buildDir.resolve("js-ir-runtime"))
archiveFileName.set("full-runtime.klib")
}
val generateReducedRuntimeKLib by eagerTask<NoDebugJavaExec> {
dependsOn(reducedRuntimeSources)
val outPath = buildDir.resolve("reducedRuntime/klib").absolutePath
buildKLib(sources = listOf(reducedRuntimeSources.get().outputs.files.singleFile.path),
dependencies = emptyList(),
outPath = outPath,
commonSources = listOf("common", "src", "unsigned").map { "$buildDir/reducedRuntime/src/libraries/stdlib/$it" }
)
}
val generateWasmRuntimeKLib by eagerTask<NoDebugJavaExec> {
buildKLib(sources = listOf("$rootDir/libraries/stdlib/wasm"),
dependencies = emptyList(),
outPath = "$buildDir/wasmRuntime/klib",
commonSources = emptyList()
)
}
val kotlinTestCommonSources = listOf(
"$rootDir/libraries/kotlin.test/annotations-common/src/main",
"$rootDir/libraries/kotlin.test/common/src/main"
)
val generateKotlinTestKLib by eagerTask<NoDebugJavaExec> {
dependsOn(generateFullRuntimeKLib)
buildKLib(
sources = listOf("$rootDir/libraries/kotlin.test/js/src/main") + kotlinTestCommonSources,
dependencies = listOf("${generateFullRuntimeKLib.outputs.files.singleFile.path}/klib"),
outPath = "$buildDir/kotlin.test/klib",
commonSources = kotlinTestCommonSources
)
}
val jsTestDir = "${buildDir}/testSrc"
val prepareStdlibTestSources by task<Sync> {
from("$rootDir/libraries/stdlib/test") {
exclude("src/generated/**")
into("test")
}
from("$rootDir/libraries/stdlib/common/test") {
exclude("src/generated/**")
into("common")
}
from("$rootDir/libraries/stdlib/js/test") {
into("js")
}
into(jsTestDir)
}
fun JavaExec.buildJs(sources: List<String>, dependencies: List<String>, outPath: String, commonSources: List<String>) {
inputs.files(sources)
outputs.dir(file(outPath).parent)
classpath = fullJsIrCli
main = "org.jetbrains.kotlin.cli.js.K2JsIrCompiler"
workingDir = rootDir
val libraryString: String = dependencies.joinToString(File.pathSeparator)
val libraryArgs: List<String> = if (libraryString.isEmpty()) emptyList() else listOf<String>("-libraries", libraryString, "-Xfriend-modules=$libraryString")
val allArgs =
sources.toList() + listOf("-output", outPath) + libraryArgs + listOf(
"-Xir-produce-js",
"-Xmulti-platform",
"-Xuse-experimental=kotlin.Experimental",
"-Xuse-experimental=kotlin.contracts.ExperimentalContracts",
"-Xuse-experimental=kotlin.ExperimentalMultiplatform",
"-Xuse-experimental=kotlin.ExperimentalStdlibApi",
"-Xuse-experimental=kotlin.ExperimentalUnsignedTypes"
)
args = allArgs
dependsOn(":compiler:cli-js:jar")
passClasspathInJar()
}
val testOutputFile = "$buildDir/kotlin-stdlib-js-ir_test.js"
val tryRunFullCli by eagerTask<NoDebugJavaExec> {
dependsOn(prepareStdlibTestSources)
dependsOn(generateFullRuntimeKLib)
dependsOn(generateKotlinTestKLib)
buildJs(
sources = listOf(jsTestDir),
dependencies = listOf(
"${generateFullRuntimeKLib.outputs.files.singleFile.path}/klib",
"${generateKotlinTestKLib.outputs.files.singleFile.path}/klib"
),
outPath = testOutputFile,
commonSources = emptyList()
)
}
node {
download = true
version = "10.16.2"
nodeModulesDir = buildDir
}
val installMocha by task<NpmTask> {
setArgs(listOf("install", "mocha"))
}
val installTeamcityReporter by task<NpmTask> {
setArgs(listOf("install", "mocha-teamcity-reporter"))
}
// TODO: TEST OUTPUT FILE
// val kotlinTestTestOutputFile = "${project(':kotlin-test:kotlin-test-js').buildDir}/classes/kotlin/test/kotlin-test-js-ir_test.js"
val runMocha by task<NodeTask> {
dependsOn(installMocha)
dependsOn(tryRunFullCli)
script = file("${buildDir}/node_modules/mocha/bin/mocha")
if (project.hasProperty("teamcity")) {
dependsOn(installTeamcityReporter)
setArgs(
listOf(
"--reporter",
"mocha-teamcity-reporter",
"--reporter-options",
"topLevelSuite=stdlib-js-ir"
)
)
}
else {
setArgs(listOf("--reporter", "min"))
}
val allArgs = getArgs().toList() + listOf(testOutputFile/*, kotlinTestTestOutputFile*/)
setArgs(allArgs)
setIgnoreExitValue(rootProject.getBooleanProperty("ignoreTestFailures") ?: false)
setWorkingDir(buildDir)
}
tasks {
test {
dependsOn(runMocha)
}
}
+5 -5
View File
@@ -310,8 +310,6 @@ task libraryJarWithoutIr(type: Jar, dependsOn: compileJs) {
}
task libraryJarWithIr(type: Zip, dependsOn: libraryJarWithoutIr) {
def irStdlibJar = tasks.getByPath(":kotlin-stdlib-js-ir:jar")
dependsOn(irStdlibJar)
archiveExtension = "jar"
destinationDirectory = file("$buildDir/libs")
@@ -319,8 +317,10 @@ task libraryJarWithIr(type: Zip, dependsOn: libraryJarWithoutIr) {
duplicatesStrategy DuplicatesStrategy.FAIL
from zipTree(libraryJarWithoutIr.archiveFile)
from(zipTree(irStdlibJar.archiveFile)) {
exclude 'META-INF/*'
if (rootProject.includeStdlibJsIr) {
def irKlib = tasks.getByPath(":kotlin-stdlib-js-ir:generateFullRuntimeKLib")
dependsOn(irKlib)
from fileTree("${irKlib.outputs.files.singleFile.path}/klib")
}
}
@@ -370,7 +370,7 @@ task distSourcesJar(type: Jar) {
artifacts {
runtime libraryJarWithIr
runtime libraryJarWithoutIr
archives libraryJarWithIr
archives sourcesJar
sources sourcesJar