[JS IR] Support per-file mode and ES modules

This commit is contained in:
Svyatoslav Kuzmich
2021-02-23 15:44:06 +03:00
parent f479ac5c3a
commit 3f8dce4b53
140 changed files with 5136 additions and 609 deletions
@@ -47,11 +47,6 @@ fun main(args: Array<String>) {
model("typescript-export/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS_IR_ES6)
}
testClass<AbstractLegacyJsTypeScriptExportTest> {
model("typescript-export/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS)
}
testClass<AbstractSourceMapGenerationSmokeTest> {
model("sourcemap/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS)
}
@@ -26,6 +26,7 @@ import org.jetbrains.kotlin.idea.KotlinFileType
import org.jetbrains.kotlin.incremental.js.IncrementalDataProviderImpl
import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl
import org.jetbrains.kotlin.incremental.js.TranslationResultValue
import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity
import org.jetbrains.kotlin.ir.backend.js.ic.ICCache
import org.jetbrains.kotlin.js.JavaScript
import org.jetbrains.kotlin.js.backend.JsToStringGenerationVisitor
@@ -46,6 +47,7 @@ import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapParser
import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapSuccess
import org.jetbrains.kotlin.js.sourceMap.SourceFilePathResolver
import org.jetbrains.kotlin.js.sourceMap.SourceMap3Builder
import org.jetbrains.kotlin.js.test.engines.ExternalTool
import org.jetbrains.kotlin.js.sourceMap.SourceMapBuilderConsumer
import org.jetbrains.kotlin.js.test.utils.*
import org.jetbrains.kotlin.js.util.TextOutputImpl
@@ -153,7 +155,17 @@ abstract class BasicBoxTest(
if (errorPolicyMatcher.find()) ErrorTolerancePolicy.resolvePolicy(errorPolicyMatcher.group(1)) else ErrorTolerancePolicy.DEFAULT
val skipDceDriven = SKIP_DCE_DRIVEN.matcher(fileContent).find()
val esModules = ES_MODULES.matcher(fileContent).find()
val splitPerModule = SPLIT_PER_MODULE.matcher(fileContent).find()
val splitPerFile = SPLIT_PER_FILE.matcher(fileContent).find()
val granularity = when {
splitPerModule -> JsGenerationGranularity.PER_MODULE
splitPerFile -> JsGenerationGranularity.PER_FILE
else -> JsGenerationGranularity.WHOLE_PROGRAM
}
val skipMangleVerification = SKIP_MANGLE_VERIFICATION.matcher(fileContent).find()
val safeExternalBoolean = SAFE_EXTERNAL_BOOLEAN.matcher(fileContent).find()
@@ -196,6 +208,12 @@ abstract class BasicBoxTest(
val mainModule = modules[mainModuleName] ?: error("No module with name \"$mainModuleName\"")
val icCache = mutableMapOf<String, TestModuleCache>()
if (esModules) {
modules.forEach { _, m -> m.moduleKind = ModuleKind.ES }
}
val customTestModule: String? = inputFiles.find { it.testEntryEsModule }?.let { File(it.fileName).readText() }
val generatedJsFiles = orderedModules.asReversed().mapNotNull { module ->
val dependencies = module.dependenciesSymbols.map { modules[it]?.outputFileName(outputDir) + ".meta.js" }
val allDependencies = module.allTransitiveDependencies().map { modules[it]?.outputFileName(outputDir) + ".meta.js" }
@@ -229,7 +247,8 @@ abstract class BasicBoxTest(
isMainModule,
expectActualLinker,
skipDceDriven,
splitPerModule,
esModules,
granularity,
errorPolicy,
propertyLazyInitialization = true,
safeExternalBoolean,
@@ -237,7 +256,8 @@ abstract class BasicBoxTest(
skipMangleVerification,
abiVersion,
checkIC,
icCache
icCache,
customTestModule,
)
when {
@@ -279,6 +299,7 @@ abstract class BasicBoxTest(
}
val additionalFiles = mutableListOf<String>()
val moduleEmulationFiles = mutableListOf<String>()
val moduleKindMatcher = MODULE_KIND_PATTERN.matcher(fileContent)
val moduleKind = if (moduleKindMatcher.find()) ModuleKind.valueOf(moduleKindMatcher.group(1)) else ModuleKind.PLAIN
@@ -286,7 +307,7 @@ abstract class BasicBoxTest(
val withModuleSystem = moduleKind != ModuleKind.PLAIN && !NO_MODULE_SYSTEM_PATTERN.matcher(fileContent).find()
if (withModuleSystem) {
additionalFiles += MODULE_EMULATION_FILE
moduleEmulationFiles += MODULE_EMULATION_FILE
}
val additionalJsFile = filePath.removeSuffix("." + KotlinFileType.EXTENSION) + JavaScript.DOT_EXTENSION
@@ -294,6 +315,63 @@ abstract class BasicBoxTest(
additionalFiles += additionalJsFile
}
if (esModules) {
val additionalMainJsFile =
(filePath.removeSuffix("." + KotlinFileType.EXTENSION) + "__main.js").takeIf { File(it).exists() }
val maybeAdditionalMjsFile: String? =
(filePath.removeSuffix("." + KotlinFileType.EXTENSION) + ".mjs")
.takeIf { File(it).exists() }
val allMjsFiles: List<String> =
inputFiles.filter { it.fileName.endsWith(".mjs") }.map { it.fileName } + listOfNotNull(maybeAdditionalMjsFile)
val allNonEsModuleFiles: List<String> =
additionalFiles + inputJsFilesBefore + globalCommonFiles + localCommonFiles + additionalCommonFiles
fun runIrEsmTests(testOutputDir: File) {
val esmOutputDir = testOutputDir.esModulesSubDir
// Copy __main file if present
if (additionalMainJsFile != null) {
val newFileName = File(esmOutputDir, "test.mjs")
newFileName.delete()
File(additionalMainJsFile).copyTo(newFileName)
}
// Copy all .mjs files into generated directory
allMjsFiles.forEach { mjsFile ->
val outFile = File(esmOutputDir, File(mjsFile).name)
File(mjsFile).copyTo(outFile)
}
val perFileEsModuleFile = "$esmOutputDir/test.mjs"
v8tool.run(*allNonEsModuleFiles.toTypedArray(), perFileEsModuleFile, *inputJsFilesAfter.toTypedArray())
}
fun File.getTestDir(): File =
File(generatedJsFiles.single().first.replace(outputDir.absolutePath, this.absolutePath))
if (!skipRegularMode) {
runIrEsmTests(outputDir.getTestDir())
if (runIrDce) {
runIrEsmTests(dceOutputDir.getTestDir())
}
}
if (runIrPir && !skipDceDriven) {
runIrEsmTests(pirOutputDir.getTestDir())
}
}
if (esModules)
return
// Old systems tests
additionalFiles.addAll(0, moduleEmulationFiles)
val additionalMainFiles = mutableListOf<String>()
val additionalMainJsFile = filePath.removeSuffix("." + KotlinFileType.EXTENSION) + "__main.js"
if (File(additionalMainJsFile).exists()) {
@@ -318,7 +396,6 @@ abstract class BasicBoxTest(
} +
globalCommonFiles + localCommonFiles + additionalCommonFiles + additionalMainFiles + inputJsFilesAfter
val dontRunGeneratedCode =
InTextDirectivesUtils.dontRunGeneratedCode(targetBackend, file)
@@ -501,7 +578,8 @@ abstract class BasicBoxTest(
isMainModule: Boolean,
expectActualLinker: Boolean,
skipDceDriven: Boolean,
splitPerModule: Boolean,
esModules: Boolean,
granularity: JsGenerationGranularity,
errorIgnorancePolicy: ErrorTolerancePolicy,
propertyLazyInitialization: Boolean,
safeExternalBoolean: Boolean,
@@ -509,7 +587,8 @@ abstract class BasicBoxTest(
skipMangleVerification: Boolean,
abiVersion: KotlinAbiVersion,
checkIC: Boolean,
icCache: MutableMap<String, TestModuleCache>
icCache: MutableMap<String, TestModuleCache>,
customTestModule: String?,
) {
val kotlinFiles = module.files.filter { it.fileName.endsWith(".kt") }
val testFiles = kotlinFiles.map { it.fileName }
@@ -558,7 +637,8 @@ abstract class BasicBoxTest(
needsFullIrRuntime,
isMainModule,
skipDceDriven,
splitPerModule,
esModules,
granularity,
propertyLazyInitialization,
safeExternalBoolean,
safeExternalBooleanDiagnostic,
@@ -566,7 +646,8 @@ abstract class BasicBoxTest(
abiVersion,
checkIC,
recompile = false,
icCache
icCache,
customTestModule,
)
if (incrementalCompilationChecksEnabled && module.hasFilesToRecompile) {
@@ -661,7 +742,8 @@ abstract class BasicBoxTest(
needsFullIrRuntime,
isMainModule = false,
skipDceDriven = true,
splitPerModule = false,
esModules = false,
granularity = JsGenerationGranularity.WHOLE_PROGRAM,
propertyLazyInitialization = true,
safeExternalBoolean = false,
safeExternalBooleanDiagnostic = null,
@@ -669,7 +751,8 @@ abstract class BasicBoxTest(
abiVersion = KotlinAbiVersion.CURRENT,
incrementalCompilation = true,
recompile = true,
mutableMapOf()
mutableMapOf(),
customTestModule = null,
)
val originalOutput = FileUtil.loadFile(outputFile)
@@ -747,7 +830,8 @@ abstract class BasicBoxTest(
needsFullIrRuntime: Boolean,
isMainModule: Boolean,
skipDceDriven: Boolean,
splitPerModule: Boolean,
esModules: Boolean,
granularity: JsGenerationGranularity,
propertyLazyInitialization: Boolean,
safeExternalBoolean: Boolean,
safeExternalBooleanDiagnostic: RuntimeDiagnostic?,
@@ -755,7 +839,8 @@ abstract class BasicBoxTest(
abiVersion: KotlinAbiVersion,
incrementalCompilation: Boolean,
recompile: Boolean,
icCache: MutableMap<String, TestModuleCache>
icCache: MutableMap<String, TestModuleCache>,
customTestModule: String?,
) {
val translator = K2JSTranslator(config, false)
val translationResult = translator.translateUnits(ExceptionThrowingReporter, units, mainCallParameters)
@@ -813,6 +898,8 @@ abstract class BasicBoxTest(
"$content\n"
ModuleKind.PLAIN -> content
ModuleKind.ES -> error("Module emulation markers are not supported for ES modules")
}
}
@@ -1082,7 +1169,8 @@ abstract class BasicBoxTest(
temporaryFile.absolutePath,
currentModule,
recompile = RECOMPILE_PATTERN.matcher(text).find(),
packageName = ktFile.packageFqName.asString()
packageName = ktFile.packageFqName.asString(),
testEntryEsModule = ENTRY_ES_MODULE.matcher(text).find(),
)
}
@@ -1099,7 +1187,7 @@ abstract class BasicBoxTest(
}
}
protected class TestFile(val fileName: String, val module: TestModule, val recompile: Boolean, val packageName: String) {
protected class TestFile(val fileName: String, val module: TestModule, val recompile: Boolean, val packageName: String, val testEntryEsModule: Boolean) {
init {
module.files += this
}
@@ -1149,6 +1237,8 @@ abstract class BasicBoxTest(
// Run top level box function
private val RUN_PLAIN_BOX_FUNCTION = Pattern.compile("^// *RUN_PLAIN_BOX_FUNCTION", Pattern.MULTILINE)
private val ENTRY_ES_MODULE = Pattern.compile("^// *ENTRY_ES_MODULE", Pattern.MULTILINE)
private val NO_INLINE_PATTERN = Pattern.compile("^// *NO_INLINE *$", Pattern.MULTILINE)
private val SKIP_NODE_JS = Pattern.compile("^// *SKIP_NODE_JS *$", Pattern.MULTILINE)
private val SKIP_MINIFICATION = Pattern.compile("^// *SKIP_MINIFICATION *$", Pattern.MULTILINE)
@@ -1163,7 +1253,10 @@ abstract class BasicBoxTest(
private val WITH_STDLIB = Pattern.compile("^// *WITH_STDLIB *\$", Pattern.MULTILINE)
private val EXPECT_ACTUAL_LINKER = Pattern.compile("^// EXPECT_ACTUAL_LINKER *$", Pattern.MULTILINE)
private val SKIP_DCE_DRIVEN = Pattern.compile("^// *SKIP_DCE_DRIVEN *$", Pattern.MULTILINE)
private val ES_MODULES = Pattern.compile("^// *ES_MODULES *$", Pattern.MULTILINE)
private val SPLIT_PER_MODULE = Pattern.compile("^// *SPLIT_PER_MODULE *$", Pattern.MULTILINE)
private val SPLIT_PER_FILE = Pattern.compile("^// *SPLIT_PER_FILE *$", Pattern.MULTILINE)
private val SKIP_MANGLE_VERIFICATION = Pattern.compile("^// *SKIP_MANGLE_VERIFICATION *$", Pattern.MULTILINE)
private val ERROR_POLICY_PATTERN = Pattern.compile("^// *ERROR_POLICY: *(.+)$", Pattern.MULTILINE)
@@ -1216,4 +1309,6 @@ fun RuntimeDiagnostic.Companion.resolve(
else -> {
null
}
}
}
val v8tool by lazy { ExternalTool(System.getProperty("javascript.engine.path.V8")) }
@@ -10,7 +10,14 @@ import org.jetbrains.kotlin.backend.common.phaser.AnyNamedPhase
import org.jetbrains.kotlin.backend.common.phaser.PhaseConfig
import org.jetbrains.kotlin.backend.common.phaser.toPhaseMap
import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport
import org.jetbrains.kotlin.config.CommonConfigurationKeys
import org.jetbrains.kotlin.ir.backend.js.*
import org.jetbrains.kotlin.ir.backend.js.codegen.CompilerOutputSink
import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity
import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity.*
import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationOptions
import org.jetbrains.kotlin.ir.backend.js.codegen.generateEsModules
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformer
import org.jetbrains.kotlin.ir.backend.js.ic.*
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl
import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrFactory
@@ -205,7 +212,8 @@ abstract class BasicIrBoxTest(
needsFullIrRuntime: Boolean,
isMainModule: Boolean,
skipDceDriven: Boolean,
splitPerModule: Boolean,
esModules: Boolean,
granularity: JsGenerationGranularity,
propertyLazyInitialization: Boolean,
safeExternalBoolean: Boolean,
safeExternalBooleanDiagnostic: RuntimeDiagnostic?,
@@ -213,7 +221,8 @@ abstract class BasicIrBoxTest(
abiVersion: KotlinAbiVersion,
incrementalCompilation: Boolean,
recompile: Boolean,
icCache: MutableMap<String, TestModuleCache>
icCache: MutableMap<String, TestModuleCache>,
customTestModule: String?,
) {
val filesToCompile = units.mapNotNull { (it as? TranslationUnit.SourceFile)?.file }
@@ -299,38 +308,73 @@ abstract class BasicIrBoxTest(
icCache = emptyMap()
)
if (!skipRegularMode) {
// val dirtyFilesToRecompile = if (recompile) {
// units.map { (it as TranslationUnit.SourceFile).file.virtualFilePath }.toSet()
// } else null
val dirtyFilesToRecompile: Set<String>? = null
val compiledModule = compile(
val mainArguments = mainCallParameters.run { if (shouldBeGenerated()) arguments() else null }
fun compileToLoweredIr(
dceDriven: Boolean,
granularity: JsGenerationGranularity,
dirtyFilesToRecompile: Set<String>? = null
): LoweredIr =
compile(
module,
phaseConfig = phaseConfig,
irFactory = IrFactoryImpl,
mainArguments = mainCallParameters.run { if (shouldBeGenerated()) arguments() else null },
exportedDeclarations = setOf(FqName.fromSegments(listOfNotNull(testPackage, testFunction))),
generateFullJs = true,
generateDceJs = runIrDce,
dceDriven = false,
dceDriven = dceDriven,
es6mode = runEs6Mode,
multiModule = splitPerModule || perModule,
propertyLazyInitialization = propertyLazyInitialization,
verifySignatures = !skipMangleVerification,
lowerPerModule = lowerPerModule,
safeExternalBoolean = safeExternalBoolean,
safeExternalBooleanDiagnostic = safeExternalBooleanDiagnostic,
verifySignatures = !skipMangleVerification,
granularity = granularity,
filesToLower = dirtyFilesToRecompile
)
// if (incrementalCompilation) {
// // TODO: enable once incremental js generation is done
// generateJsFromAst(klibPath, icCache.map { it.key to it.value.createModuleCache() }.toMap())
// }
fun generateTestFile(outputDir: File) {
val moduleName = config.configuration[CommonConfigurationKeys.MODULE_NAME]
val esmTestFile = File(outputDir, "test.mjs")
logger.logFile("ES module test file", esmTestFile)
val defaultTestModule =
"""
import { box } from './${moduleName}/index.js';
let res = box();
if (res !== "OK") {
throw "Wrong result: " + String(res);
}
""".trimIndent()
val jsOutputFile = if (recompile) File(outputFile.parentFile, outputFile.nameWithoutExtension + "-recompiled.js")
else outputFile
esmTestFile.writeText(customTestModule ?: defaultTestModule)
}
val options = JsGenerationOptions(generatePackageJson = true, generateTypeScriptDefinitions = generateDts)
fun generateEsModules(ir: LoweredIr, outputDir: File, granularity: JsGenerationGranularity) {
outputDir.deleteRecursively()
generateEsModules(ir, jsOutputSink(outputDir), mainArguments = mainArguments, granularity = granularity, options = options)
generateTestFile(outputDir)
}
fun generateOldModuleSystems(
ir: LoweredIr,
outputFile: File,
outputDceFile: File,
granularity: JsGenerationGranularity,
runDce: Boolean,
dirtyFilesToRecompile: Set<String>? = null
) {
outputFile.deleteRecursively()
check(granularity != PER_FILE) { "Per file granularity is not supported for old module systems" }
val transformer = IrModuleToJsTransformer(
ir.context,
mainArguments,
fullJs = true,
dceJs = runDce,
multiModule = granularity == PER_MODULE,
relativeRequirePath = false
)
val compiledModule: CompilerResult = transformer.generateModule(ir.allModules)
val compiledOutput = if (dirtyFilesToRecompile != null) CompilationOutputs(
"""
@@ -343,9 +387,9 @@ abstract class BasicIrBoxTest(
) else compiledModule.outputs!!
val compiledDCEOutput = if (dirtyFilesToRecompile != null) null else compiledModule.outputsAfterDce
compiledOutput.writeTo(jsOutputFile, config)
compiledOutput.writeTo(outputFile, config)
compiledDCEOutput?.writeTo(dceOutputFile, config)
compiledDCEOutput?.writeTo(outputDceFile, config)
if (generateDts) {
val dtsFile = outputFile.withReplacedExtensionOrNull("_v5.js", ".d.ts")!!
@@ -356,26 +400,31 @@ abstract class BasicIrBoxTest(
compiledOutput.jsProgram?.let { processJsProgram(it, units) }
}
if (runIrPir) {
val compiledModule = compile(
module,
phaseConfig = phaseConfig,
irFactory = PersistentIrFactory(),
mainArguments = mainCallParameters.run { if (shouldBeGenerated()) arguments() else null },
exportedDeclarations = setOf(FqName.fromSegments(listOfNotNull(testPackage, testFunction))),
generateFullJs = true,
generateDceJs = runIrDce,
dceDriven = true,
es6mode = runEs6Mode,
multiModule = splitPerModule || perModule,
propertyLazyInitialization = propertyLazyInitialization,
lowerPerModule = lowerPerModule,
safeExternalBoolean = safeExternalBoolean,
safeExternalBooleanDiagnostic = safeExternalBooleanDiagnostic,
verifySignatures = !skipMangleVerification,
)
compiledModule.outputs!!.writeTo(pirOutputFile, config)
processJsProgram(compiledModule.outputs!!.jsProgram!!, units)
@Suppress("NAME_SHADOWING")
val granularity = if (perModule) PER_MODULE else granularity
if (!skipRegularMode) {
val ir = compileToLoweredIr(dceDriven = false, granularity)
if (esModules) {
generateEsModules(ir, outputFile.esModulesSubDir, granularity)
if (runIrDce) {
eliminateDeadDeclarations(ir.allModules, ir.context)
generateEsModules(ir, dceOutputFile.esModulesSubDir, granularity)
}
} else {
val jsOutputFile = if (recompile) File(outputFile.parentFile, outputFile.nameWithoutExtension + "-recompiled.js")
else outputFile
generateOldModuleSystems(ir, jsOutputFile, dceOutputFile, granularity, runIrDce)
}
}
if (runIrPir && !skipDceDriven) {
val ir = compileToLoweredIr(dceDriven = true, granularity)
if (esModules) {
generateEsModules(ir, pirOutputFile.esModulesSubDir, granularity)
} else {
generateOldModuleSystems(ir, pirOutputFile, pirOutputFile, granularity, false)
}
}
}
}
@@ -453,15 +502,19 @@ abstract class BasicIrBoxTest(
needsFullIrRuntime,
isMainModule,
skipDceDriven = true,
splitPerModule = false, // TODO??
granularity = WHOLE_PROGRAM, // TODO??
propertyLazyInitialization = true,
safeExternalBoolean = false,
safeExternalBooleanDiagnostic = null,
skipMangleVerification = false,
KotlinAbiVersion.CURRENT,
abiVersion = KotlinAbiVersion.CURRENT,
incrementalCompilation = true,
recompile = true,
icCaches
icCache = icCaches,
// TODO??
customTestModule = null,
esModules = false,
)
val cacheProvider = icCache.cacheProvider()
@@ -481,6 +534,19 @@ abstract class BasicIrBoxTest(
}
}
private fun jsOutputSink(perFileOutputDir: File): CompilerOutputSink {
perFileOutputDir.deleteRecursively()
perFileOutputDir.mkdirs()
return object : CompilerOutputSink {
override fun write(module: String, path: String, content: String) {
val file = File(File(perFileOutputDir, module), path)
file.parentFile.mkdirs()
file.writeText(content)
}
}
}
private fun createIcCache(
path: String,
dirtyFiles: Collection<String>?,
@@ -561,8 +627,10 @@ abstract class BasicIrBoxTest(
}
}
private fun File.write(text: String) {
parentFile.mkdirs()
writeText(text)
}
val File.esModulesSubDir: File
get() = File(absolutePath + "_esm")
@@ -7,8 +7,12 @@ package org.jetbrains.kotlin.js.test.engines
import java.io.BufferedReader
import java.io.InputStreamReader
import java.lang.Boolean.getBoolean
import kotlin.test.fail
val toolLogsEnabled: Boolean = getBoolean("kotlin.js.test.verbose")
class ExternalTool(val path: String) {
fun run(vararg arguments: String) {
val command = arrayOf(path, *arguments)
@@ -17,7 +21,9 @@ class ExternalTool(val path: String) {
.start()
val commandString = command.joinToString(" ") { escapeShellArgument(it) }
println(commandString)
if (toolLogsEnabled) {
println(commandString)
}
// Print process output
val input = BufferedReader(InputStreamReader(process.inputStream))
@@ -1,13 +0,0 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.js.test.semantics
import org.jetbrains.kotlin.js.test.BasicBoxTest
abstract class AbstractLegacyJsTypeScriptExportTest : BasicBoxTest(
pathToTestDir = TEST_DATA_DIR_PATH + "typescript-export/",
testGroupOutputDirPrefix = "legacy-typescript-export/"
)