[JS IR] Build stdlib using compiler from repo

Previously JS IR versions of stdlib and kotlin-test were build by default using compiler previously built on a buildserver. It had some issues: - This required us to advance bootstrap every time we made any incompatible IR changes. This happens often since IR ABI is not quite stable yet. - We never tested the exact combination of compiler and stdlib we publish We tested: - new compiler with new stdlib build by new compiler (in box tests) - old compiler with new stdlib build by old compiler (in stdlib tests) We published: - new compiler with new stdlib build by old compiler After this change JS IR compiler tests, builds and publishes single configuration: new compiler with new stdlib build by new compiler JS IR stdlib and kotlin-test are now built using JavaExec of CLI instead of Gradle plugin to avoid troubles of loading a freshly built plugin. This also allows to have a granular dependencies: we don't rebuild klib if we changed a lowering in a compiler backend, but we do rebuild it if we changed IR serialization algorithm.
2019-11-18 20:17:21 +03:00
parent 70111f8db1
commit 31c84e9ac4
20 changed files with 439 additions and 420 deletions
@@ -26,197 +26,4 @@ sourceSets {
    "test" { projectDefault() }
 }

-val unimplementedNativeBuiltIns =
-    (file("$rootDir/core/builtins/native/kotlin/").list().toSet() - file("$rootDir/libraries/stdlib/js-ir/builtins/").list())
-        .map { "core/builtins/native/kotlin/$it" }
-
-// Required to compile native builtins with the rest of runtime
-val builtInsHeader = """@file:Suppress(
-    "NON_ABSTRACT_FUNCTION_WITH_NO_BODY",
-    "MUST_BE_INITIALIZED_OR_BE_ABSTRACT",
-    "EXTERNAL_TYPE_EXTENDS_NON_EXTERNAL_TYPE",
-    "PRIMARY_CONSTRUCTOR_DELEGATION_CALL_EXPECTED",
-    "WRONG_MODIFIER_TARGET"
-)
-"""
-
-val fullRuntimeSources by task<Sync> {
-
-    val sources = listOf(
-        "core/builtins/src/kotlin/",
-        "libraries/stdlib/common/src/",
-        "libraries/stdlib/src/kotlin/",
-        "libraries/stdlib/unsigned/",
-        "libraries/stdlib/js/src/",
-        "libraries/stdlib/js/runtime/",
-        "libraries/stdlib/js-ir/builtins/",
-        "libraries/stdlib/js-ir/src/",
-        "libraries/stdlib/js-ir/runtime/",
-
-        // TODO get rid - move to test module
-        "js/js.translator/testData/_commonFiles/"
-    ) + unimplementedNativeBuiltIns
-
-    val excluded = listOf(
-        // stdlib/js/src/generated is used exclusively for current `js-v1` backend.
-        "libraries/stdlib/js/src/generated/**",
-
-        // JS-specific optimized version of emptyArray() already defined
-        "core/builtins/src/kotlin/ArrayIntrinsics.kt"
-    )
-
-    sources.forEach { path ->
-        from("$rootDir/$path") {
-            into(path.dropLastWhile { it != '/' })
-            excluded.filter { it.startsWith(path) }.forEach {
-                exclude(it.substring(path.length))
-            }
-        }
-    }
-
-    into("$buildDir/fullRuntime/src")
-
-    doLast {
-        unimplementedNativeBuiltIns.forEach { path ->
-            val file = File("$buildDir/fullRuntime/src/$path")
-            val sourceCode = builtInsHeader + file.readText()
-            file.writeText(sourceCode)
-        }
-    }
-}
-
-val reducedRuntimeSources by task<Sync> {
-    dependsOn(fullRuntimeSources)
-
-    from(fullRuntimeSources.get().outputs.files.singleFile) {
-        exclude(
-            listOf(
-                "libraries/stdlib/unsigned/**",
-                "libraries/stdlib/common/src/generated/_Arrays.kt",
-                "libraries/stdlib/common/src/generated/_Collections.kt",
-                "libraries/stdlib/common/src/generated/_Comparisons.kt",
-                "libraries/stdlib/common/src/generated/_Maps.kt",
-                "libraries/stdlib/common/src/generated/_Sequences.kt",
-                "libraries/stdlib/common/src/generated/_Sets.kt",
-                "libraries/stdlib/common/src/generated/_Strings.kt",
-                "libraries/stdlib/common/src/generated/_UArrays.kt",
-                "libraries/stdlib/common/src/generated/_URanges.kt",
-                "libraries/stdlib/common/src/generated/_UCollections.kt",
-                "libraries/stdlib/common/src/generated/_UComparisons.kt",
-                "libraries/stdlib/common/src/generated/_USequences.kt",
-                "libraries/stdlib/common/src/kotlin/SequencesH.kt",
-                "libraries/stdlib/common/src/kotlin/TextH.kt",
-                "libraries/stdlib/common/src/kotlin/UMath.kt",
-                "libraries/stdlib/common/src/kotlin/collections/**",
-                "libraries/stdlib/common/src/kotlin/ioH.kt",
-                "libraries/stdlib/js-ir/runtime/collectionsHacks.kt",
-                "libraries/stdlib/js-ir/src/generated/**",
-                "libraries/stdlib/js-ir/src/kotlin/text/**",
-                "libraries/stdlib/js/src/jquery/**",
-                "libraries/stdlib/js/src/org.w3c/**",
-                "libraries/stdlib/js/src/kotlin/char.kt",
-                "libraries/stdlib/js/src/kotlin/collections.kt",
-                "libraries/stdlib/js/src/kotlin/collections/**",
-                "libraries/stdlib/js/src/kotlin/time/**",
-                "libraries/stdlib/js/src/kotlin/console.kt",
-                "libraries/stdlib/js/src/kotlin/coreDeprecated.kt",
-                "libraries/stdlib/js/src/kotlin/date.kt",
-                "libraries/stdlib/js/src/kotlin/debug.kt",
-                "libraries/stdlib/js/src/kotlin/grouping.kt",
-                "libraries/stdlib/js/src/kotlin/json.kt",
-                "libraries/stdlib/js/src/kotlin/promise.kt",
-                "libraries/stdlib/js/src/kotlin/regexp.kt",
-                "libraries/stdlib/js/src/kotlin/sequence.kt",
-                "libraries/stdlib/js/src/kotlin/text/**",
-                "libraries/stdlib/js/src/kotlin/reflect/KTypeHelpers.kt",
-                "libraries/stdlib/js/src/kotlin/reflect/KTypeParameterImpl.kt",
-                "libraries/stdlib/js/src/kotlin/reflect/KTypeImpl.kt",
-                "libraries/stdlib/src/kotlin/collections/**",
-                "libraries/stdlib/src/kotlin/experimental/bitwiseOperations.kt",
-                "libraries/stdlib/src/kotlin/properties/Delegates.kt",
-                "libraries/stdlib/src/kotlin/random/URandom.kt",
-                "libraries/stdlib/src/kotlin/text/**",
-                "libraries/stdlib/src/kotlin/time/**",
-                "libraries/stdlib/src/kotlin/util/KotlinVersion.kt",
-                "libraries/stdlib/src/kotlin/util/Tuples.kt",
-                "libraries/stdlib/js/src/kotlin/dom/**",
-                "libraries/stdlib/js/src/kotlin/browser/**"
-            )
-        )
-    }
-
-    from("$rootDir/libraries/stdlib/js-ir/smallRuntime") {
-        into("libraries/stdlib/js-ir/runtime/")
-    }
-
-    into("$buildDir/reducedRuntime/src")
-}
-
-
-fun JavaExec.buildKLib(sources: List<String>, dependencies: List<String>, outPath: String, commonSources: List<String>) {
-    inputs.files(sources)
-    outputs.dir(file(outPath).parent)
-
-    classpath = sourceSets.test.get().runtimeClasspath
-    main = "org.jetbrains.kotlin.ir.backend.js.GenerateIrRuntimeKt"
-    workingDir = rootDir
-    args = sources.toList() + listOf("-o", outPath) + dependencies.flatMap { listOf("-d", it) } + commonSources.flatMap { listOf("-c", it) }
-
-    passClasspathInJar()
-}
-
-val fullRuntimeDir = buildDir.resolve("fullRuntime/klib")
-
-val generateFullRuntimeKLib by eagerTask<NoDebugJavaExec> {
-    dependsOn(fullRuntimeSources)
-
-    buildKLib(sources = listOf(fullRuntimeSources.get().outputs.files.singleFile.path),
-              dependencies = emptyList(),
-              outPath = fullRuntimeDir.absolutePath,
-              commonSources = listOf("common", "src", "unsigned").map { "$buildDir/fullRuntime/src/libraries/stdlib/$it" }
-    )
-}
-
-val packFullRuntimeKLib by tasks.registering(Jar::class) {
-    dependsOn(generateFullRuntimeKLib)
-    from(fullRuntimeDir)
-    destinationDirectory.set(rootProject.buildDir.resolve("js-ir-runtime"))
-    archiveFileName.set("full-runtime.klib")
-}
-
-val generateReducedRuntimeKLib by eagerTask<NoDebugJavaExec> {
-    dependsOn(reducedRuntimeSources)
-
-    val outPath = buildDir.resolve("reducedRuntime/klib").absolutePath
-    buildKLib(sources = listOf(reducedRuntimeSources.get().outputs.files.singleFile.path),
-              dependencies = emptyList(),
-              outPath = outPath,
-              commonSources = listOf("common", "src", "unsigned").map { "$buildDir/reducedRuntime/src/libraries/stdlib/$it" }
-    )
-}
-
-val generateWasmRuntimeKLib by eagerTask<NoDebugJavaExec> {
-    buildKLib(sources = listOf("$rootDir/libraries/stdlib/wasm"),
-              dependencies = emptyList(),
-              outPath = "$buildDir/wasmRuntime/klib",
-              commonSources = emptyList()
-    )
-}
-
-
-val kotlinTestCommonSources = listOf(
-    "$rootDir/libraries/kotlin.test/annotations-common/src/main",
-    "$rootDir/libraries/kotlin.test/common/src/main"
-)
-val generateKotlinTestKLib by eagerTask<NoDebugJavaExec> {
-    dependsOn(generateFullRuntimeKLib)
-
-    buildKLib(
-        sources = listOf("$rootDir/libraries/kotlin.test/js/src/main") + kotlinTestCommonSources,
-        dependencies = listOf("${generateFullRuntimeKLib.outputs.files.singleFile.path}/klib"),
-        outPath = "$buildDir/kotlin.test/klib",
-        commonSources = kotlinTestCommonSources
-    )
-}
-
 testsJar {}
@@ -19,6 +19,8 @@ import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
 import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.cli.common.messages.MessageRenderer
+import org.jetbrains.kotlin.cli.common.messages.PrintingMessageCollector
 import org.jetbrains.kotlin.config.*
 import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
@@ -297,8 +299,10 @@ private class ModulesStructure(
    val builtInsDep = allDependencies.getFullList().find { it.isBuiltIns }

    fun runAnalysis(): JsAnalysisResult {
+        // TODO: Should we not provide default message collector?
        val messageCollector: MessageCollector =
            compilerConfiguration.get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY) ?: MessageCollector.NONE
+
        val analyzerWithCompilerReport = AnalyzerWithCompilerReport(
            messageCollector,
            compilerConfiguration.languageVersionSettings
@@ -1,137 +0,0 @@
-/*
- * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.ir.backend.js
-
-import com.intellij.openapi.Disposable
-import com.intellij.openapi.vfs.StandardFileSystems
-import com.intellij.openapi.vfs.VirtualFileManager
-import com.intellij.psi.PsiManager
-import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
-import org.jetbrains.kotlin.cli.common.messages.GroupingMessageCollector
-import org.jetbrains.kotlin.cli.common.messages.MessageCollector
-import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
-import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
-import org.jetbrains.kotlin.config.*
-import org.jetbrains.kotlin.js.config.JSConfigurationKeys
-import org.jetbrains.kotlin.library.resolver.KotlinLibraryResolveResult
-import org.jetbrains.kotlin.psi.KtFile
-import org.jetbrains.kotlin.resolve.multiplatform.isCommonSource
-import org.jetbrains.kotlin.serialization.js.ModuleKind
-import org.jetbrains.kotlin.util.Logger
-import java.io.File
-
-fun buildConfiguration(environment: KotlinCoreEnvironment, moduleName: String): CompilerConfiguration {
-    val runtimeConfiguration = environment.configuration.copy()
-    runtimeConfiguration.put(CommonConfigurationKeys.MODULE_NAME, moduleName)
-    runtimeConfiguration.put(JSConfigurationKeys.MODULE_KIND, ModuleKind.PLAIN)
-
-    runtimeConfiguration.languageVersionSettings = LanguageVersionSettingsImpl(
-        LanguageVersion.LATEST_STABLE, ApiVersion.LATEST_STABLE,
-        specificFeatures = mapOf(
-            LanguageFeature.AllowContractsForCustomFunctions to LanguageFeature.State.ENABLED,
-            LanguageFeature.MultiPlatformProjects to LanguageFeature.State.ENABLED
-        ),
-        analysisFlags = mapOf(
-            AnalysisFlags.useExperimental to listOf(
-                "kotlin.contracts.ExperimentalContracts",
-                "kotlin.Experimental",
-                "kotlin.ExperimentalMultiplatform"
-            ),
-            AnalysisFlags.allowResultReturnType to true
-        )
-    )
-
-    return runtimeConfiguration
-}
-
-val environment = KotlinCoreEnvironment.createForTests(Disposable { }, CompilerConfiguration(), EnvironmentConfigFiles.JS_CONFIG_FILES)
-
-fun createPsiFile(fileName: String): KtFile {
-    val psiManager = PsiManager.getInstance(environment.project)
-    val fileSystem = VirtualFileManager.getInstance().getFileSystem(StandardFileSystems.FILE_PROTOCOL)
-
-    val file = fileSystem.findFileByPath(fileName) ?: error("File not found: $fileName")
-
-    return psiManager.findFile(file) as KtFile
-}
-
-
-fun buildKLib(
-    moduleName: String,
-    sources: List<String>,
-    outputPath: String,
-    allDependencies: KotlinLibraryResolveResult,
-    commonSources: List<String>
-) {
-    generateKLib(
-        project = environment.project,
-        files = sources.map { source ->
-            val file = createPsiFile(source)
-            if (source in commonSources) {
-                file.isCommonSource = true
-            }
-            file
-        },
-        configuration = buildConfiguration(environment, moduleName),
-        allDependencies = allDependencies,
-        friendDependencies = emptyList(),
-        outputKlibPath = outputPath,
-        nopack = true
-    )
-}
-
-private fun listOfKtFilesFrom(paths: List<String>): List<String> {
-    val currentDir = File("")
-    return paths.flatMap { path ->
-        File(path)
-            .walkTopDown()
-            .filter { it.extension == "kt" }
-            .map { it.relativeToOrSelf(currentDir).path }
-            .asIterable()
-    }.distinct()
-}
-
-fun main(args: Array<String>) {
-
-    val inputFiles = mutableListOf<String>()
-    var outputPath: String? = null
-    val dependencies = mutableListOf<String>()
-    val commonSources = mutableListOf<String>()
-
-    var index = 0
-    while (index < args.size) {
-        val arg = args[index++]
-
-        when (arg) {
-            "-o" -> outputPath = args[index++]
-            "-d" -> dependencies += args[index++]
-            "-c" -> commonSources += args[index++]
-            else -> inputFiles += arg
-        }
-    }
-
-    if (outputPath == null) {
-        error("Please set path to .klm file: `-o some/dir/module-name.klm`")
-    }
-
-    val resolvedLibraries = jsResolveLibraries(
-        dependencies, messageCollectorLogger(MessageCollector.NONE)
-    )
-
-    buildKLib(File(outputPath).absolutePath, listOfKtFilesFrom(inputFiles), outputPath, resolvedLibraries, listOfKtFilesFrom(commonSources))
-}
-
-// Copied here from `K2JsIrCompiler` instead of reusing in order to avoid circular dependencies between Gradle tasks
-private fun messageCollectorLogger(collector: MessageCollector) = object : Logger {
-    override fun warning(message: String)= collector.report(CompilerMessageSeverity.STRONG_WARNING, message)
-    override fun error(message: String) = collector.report(CompilerMessageSeverity.ERROR, message)
-    override fun log(message: String) = collector.report(CompilerMessageSeverity.LOGGING, message)
-    override fun fatal(message: String): Nothing {
-        collector.report(CompilerMessageSeverity.ERROR, message)
-        (collector as? GroupingMessageCollector)?.flush()
-        kotlin.error(message)
-    }
-}