azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/JS] Add file-to-file compilation ^KT-6168 Fixed

Browse Source
This commit is contained in:
Artem Kobzar
2023-06-22 18:23:45 +00:00
committed by Space Team
parent a44b5e4562
commit 64158a8a2f
74 changed files with 947 additions and 608 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt
+2 -2
View File
@@ -23,11 +23,11 @@ import org.jetbrains.kotlin.config.languageVersionSettings
import org.jetbrains.kotlin.ir.backend.js.JsIrCompilerWithIC
import org.jetbrains.kotlin.ir.backend.js.SourceMapsInfo
import org.jetbrains.kotlin.ir.backend.js.WholeWorldStageController
import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity
import org.jetbrains.kotlin.ir.backend.js.ic.*
import org.jetbrains.kotlin.ir.backend.js.jsPhases
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.CompilationOutputs
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.extension
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsGenerationGranularity
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.safeModuleName
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC
import org.jetbrains.kotlin.js.config.JSConfigurationKeys
@@ -361,7 +361,7 @@ abstract class AbstractInvalidationTest(private val targetBackend: TargetBackend
relativeRequirePath = true
)
val (jsOutput, rebuiltModules) = jsExecutableProducer.buildExecutable(multiModule = true, outJsProgram = true)
val (jsOutput, rebuiltModules) = jsExecutableProducer.buildExecutable(granularity = JsGenerationGranularity.PER_MODULE, outJsProgram = true)
val writtenFiles = writeJsCode(projStep.id, mainModuleName, jsOutput)
verifyJsExecutableProducerBuildModules(projStep.id, rebuiltModules, projStep.dirtyJS)
js/js.tests/test/org/jetbrains/kotlin/js/test/converters/JsIrBackendFacade.kt
+27 -19
View File
@@ -13,11 +13,11 @@ import org.jetbrains.kotlin.cli.common.isWindows
import org.jetbrains.kotlin.config.CommonConfigurationKeys
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.ir.backend.js.*
import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity
import org.jetbrains.kotlin.ir.backend.js.ic.JsExecutableProducer
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerDesc
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.*
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.ir.backend.js.SourceMapsInfo
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC
import org.jetbrains.kotlin.ir.linkage.partial.PartialLinkageConfig
import org.jetbrains.kotlin.ir.linkage.partial.PartialLinkageLogLevel
@@ -54,10 +54,10 @@ class JsIrBackendFacade(
val testServices: TestServices,
private val firstTimeCompilation: Boolean
) : AbstractTestFacade<BinaryArtifacts.KLib, BinaryArtifacts.Js>() {
override val inputKind: ArtifactKinds.KLib
get() = ArtifactKinds.KLib
override val outputKind: ArtifactKinds.Js
get() = ArtifactKinds.Js
override val inputKind: ArtifactKinds.KLib get() = ArtifactKinds.KLib
override val outputKind: ArtifactKinds.Js get() = ArtifactKinds.Js
private val jsIrPathReplacer by lazy { JsIrPathReplacer(testServices) }
constructor(testServices: TestServices) : this(testServices, firstTimeCompilation = true)
@@ -90,8 +90,8 @@ class JsIrBackendFacade(
val granularity = when {
!firstTimeCompilation -> JsGenerationGranularity.WHOLE_PROGRAM
splitPerFile || module.kind == ModuleKind.ES -> JsGenerationGranularity.PER_FILE
splitPerModule || perModule -> JsGenerationGranularity.PER_MODULE
splitPerFile -> JsGenerationGranularity.PER_FILE
else -> JsGenerationGranularity.WHOLE_PROGRAM
}
@@ -116,7 +116,7 @@ class JsIrBackendFacade(
caches = testServices.jsIrIncrementalDataProvider.getCaches(),
relativeRequirePath = false
)
jsExecutableProducer.buildExecutable(it.perModule, true).compilationOut
jsExecutableProducer.buildExecutable(it.granularity, true).compilationOut
}
)
return BinaryArtifacts.Js.JsIrArtifact(
@@ -168,6 +168,7 @@ class JsIrBackendFacade(
val runIrDce = JsEnvironmentConfigurationDirectives.RUN_IR_DCE in module.directives
val onlyIrDce = JsEnvironmentConfigurationDirectives.ONLY_IR_DCE in module.directives
val perModuleOnly = JsEnvironmentConfigurationDirectives.SPLIT_PER_MODULE in module.directives
val perFileOnly = JsEnvironmentConfigurationDirectives.SPLIT_PER_FILE in module.directives
val isEsModules = JsEnvironmentConfigurationDirectives.ES_MODULES in module.directives ||
module.directives[JsEnvironmentConfigurationDirectives.MODULE_KIND].contains(ModuleKind.ES)
@@ -182,7 +183,7 @@ class JsIrBackendFacade(
mainArguments,
moduleToName = runIf(isEsModules) {
loweredIr.allModules.associateWith {
"./${getJsArtifactSimpleName(testServices, it.safeName)}_v5.mjs".minifyIfNeed()
"./${getJsArtifactSimpleName(testServices, it.safeName)}_v5".minifyIfNeed()
}
} ?: emptyMap()
)
@@ -190,17 +191,21 @@ class JsIrBackendFacade(
// If perModuleOnly then skip whole program
// (it.dce => runIrDce) && (perModuleOnly => it.perModule)
val translationModes = TranslationMode.values()
.filter { (it.production || !onlyIrDce) && (!it.production || runIrDce) && (!perModuleOnly || it.perModule) }
.filter {
(it.production || !onlyIrDce) &&
(!it.production || runIrDce) &&
(!perModuleOnly || it.granularity == JsGenerationGranularity.PER_MODULE) &&
(!perFileOnly || it.granularity == JsGenerationGranularity.PER_FILE)
}
.filter { it.production == it.minimizedMemberNames }
.filter { isEsModules || it.granularity != JsGenerationGranularity.PER_FILE }
.toSet()
val compilationOut = transformer.generateModule(loweredIr.allModules, translationModes, false)
val compilationOut = transformer.generateModule(loweredIr.allModules, translationModes, isEsModules)
return BinaryArtifacts.Js.JsIrArtifact(outputFile, compilationOut).dump(module)
}
private fun IrModuleFragment.resolveTestPaths() {
JsIrPathReplacer(testServices).let {
files.forEach(it::lower)
}
files.forEach(jsIrPathReplacer::lower)
}
private fun loadIrFromKlib(module: TestModule, configuration: CompilerConfiguration): IrModuleInfo {
@@ -268,8 +273,9 @@ class JsIrBackendFacade(
return this
}
fun File.fixJsFile(newJsTarget: File, moduleId: String, moduleKind: ModuleKind) {
fun File.fixJsFile(rootDir: File, newJsTarget: File, moduleId: String, moduleKind: ModuleKind) {
val newJsCode = ClassicJsBackendFacade.wrapWithModuleEmulationMarkers(readText(), moduleKind, moduleId)
val jsCodeWithCorrectImportPath = jsIrPathReplacer.replacePathTokensWithRealPath(newJsCode, newJsTarget, rootDir)
val oldJsMap = File("$absolutePath.map")
val jsCodeMap = (moduleKind == ModuleKind.PLAIN && oldJsMap.exists()).ifTrue { oldJsMap.readText() }
@@ -277,12 +283,13 @@ class JsIrBackendFacade(
this.delete()
oldJsMap.delete()
newJsTarget.write(newJsCode)
newJsTarget.write(jsCodeWithCorrectImportPath)
jsCodeMap?.let { File("${newJsTarget.absolutePath}.map").write(it) }
}
private fun CompilationOutputs.writeTo(outputFile: File, moduleId: String, moduleKind: ModuleKind) {
val tmpBuildDir = outputFile.parentFile.resolve("tmp-build")
val rootDir = outputFile.parentFile
val tmpBuildDir = rootDir.resolve("tmp-build")
// CompilationOutputs keeps the `outputDir` clean by removing all outdated JS and other unknown files.
// To ensure that useful files around `outputFile`, such as irdump, are not removed, use `tmpBuildDir` instead.
val allJsFiles = writeAll(tmpBuildDir, outputFile.nameWithoutExtension, false, moduleId, moduleKind).filter {
@@ -290,11 +297,11 @@ class JsIrBackendFacade(
}
val mainModuleFile = allJsFiles.last()
mainModuleFile.fixJsFile(outputFile, moduleId, moduleKind)
mainModuleFile.fixJsFile(rootDir, outputFile, moduleId, moduleKind)
dependencies.map { it.first }.zip(allJsFiles.dropLast(1)).forEach { (depModuleId, builtJsFilePath) ->
val newFile = outputFile.augmentWithModuleName(depModuleId)
builtJsFilePath.fixJsFile(newFile, depModuleId, moduleKind)
builtJsFilePath.fixJsFile(rootDir, newFile, depModuleId, moduleKind)
}
tmpBuildDir.deleteRecursively()
}
@@ -347,8 +354,9 @@ fun String.minifyIfNeed(): String {
if (fileName.length <= 80) return this
val fileExtension = fileFullName.substringAfterLast('.')
val extensionPart = if (fileExtension.isEmpty()) "" else ".$fileExtension"
return "$directoryPath$delimiter${fileName.cityHash64().toULong().toString(16)}.$fileExtension"
return "$directoryPath$delimiter${fileName.cityHash64().toULong().toString(16)}$extensionPart"
}
fun File.augmentWithModuleName(moduleName: String): File = File(absolutePath.augmentWithModuleName(moduleName))
js/js.tests/test/org/jetbrains/kotlin/js/test/converters/JsIrPathReplacer.kt
+19 -7
View File
@@ -1,5 +1,5 @@
/*
* Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
@@ -18,6 +18,10 @@ import org.jetbrains.kotlin.test.services.TestServices
import org.jetbrains.kotlin.test.services.isJsFile
import org.jetbrains.kotlin.test.services.isMjsFile
import org.jetbrains.kotlin.test.services.moduleStructure
import java.io.File
import kotlin.io.invariantSeparatorsPath
private const val PATH_TO_ROOT_TOKEN = "@PATH_TO_ROOT"
class JsIrPathReplacer(testServices: TestServices) : DeclarationTransformer {
private val replacements = testServices.collectReplacementsMap()
@@ -35,6 +39,7 @@ class JsIrPathReplacer(testServices: TestServices) : DeclarationTransformer {
private fun IrAnnotationContainer.replaceJsModulePath() {
val jsModuleAnnotation = getAnnotation(JsAnnotations.jsModuleFqn) ?: return
@Suppress("UNCHECKED_CAST")
val stringLiteral = jsModuleAnnotation.getValueArgument(0) as IrConst<String>
val pathReplacement = stringLiteral.getReplacement() ?: return
@@ -43,17 +48,24 @@ class JsIrPathReplacer(testServices: TestServices) : DeclarationTransformer {
}
private fun IrConst<String>.getReplacement(): IrConst<String>? {
val replacement = replacements[value] ?: replacements[value.replace("./", "")] ?: return null
return IrConstImpl.string(startOffset, endOffset, type, "./" + replacement.replace("./", ""))
return IrConstImpl.string(startOffset, endOffset, type, replacements[value] ?: return null)
}
private fun TestServices.collectReplacementsMap(): Map<String, String> {
return moduleStructure.modules.asSequence()
.map { module -> module to module.files.filter { it.isJsFile || it.isMjsFile } }
.filter { (_, files) -> files.isNotEmpty() }
.flatMap { (module, files) -> files.map { it.relativePath to module.getNameFor(it, this) } }
.plus(getAdditionalFiles(this).map { it.name to it.name })
.plus(getAdditionalMainFiles(this).map { it.name to it.name })
.flatMap { (module, files) -> files.map { "./${it.relativePath}" to "$PATH_TO_ROOT_TOKEN/${module.getNameFor(it, this)}" } }
.plus(getAdditionalFiles(this).map { "./${it.name}" to "$PATH_TO_ROOT_TOKEN/${it.name}" })
.plus(getAdditionalMainFiles(this).map { "./${it.name}" to "$PATH_TO_ROOT_TOKEN/${it.name}" })
.toMap()
}
}
fun replacePathTokensWithRealPath(content: String, file: File, rootDir: File): String {
val relativePathToRootDir = rootDir.relativeTo(file.parentFile).invariantSeparatorsPath.ifEmpty { "." }
return content.replace("from '$PATH_TO_ROOT_TOKEN/(.+?)';${'$'}".toRegex(RegexOption.MULTILINE)) { result ->
val importPath = result.groups[1]?.value ?: error("Unexpected import path")
"from '$relativePathToRootDir/$importPath'\n"
}
}
}
js/js.tests/test/org/jetbrains/kotlin/js/test/handlers/JsArtifactsDumpHandler.kt
+8 -2
View File
@@ -31,17 +31,23 @@ class JsArtifactsDumpHandler(testServices: TestServices) : AfterAnalysisChecker(
val testGroupOutputDirForMinification = File(pathToRootOutputDir + "out-min/" + testGroupOutputDirPrefix)
val testGroupOutputDirForPerModuleCompilation = File(pathToRootOutputDir + "out-per-module/" + testGroupOutputDirPrefix)
val testGroupOutputDirForPerModuleMinification = File(pathToRootOutputDir + "out-per-module-min/" + testGroupOutputDirPrefix)
val testGroupOutputDirForPerFileCompilation = File(pathToRootOutputDir + "out-per-file/" + testGroupOutputDirPrefix)
val testGroupOutputDirForPerFileMinification = File(pathToRootOutputDir + "out-per-file-min/" + testGroupOutputDirPrefix)
val outputDir = getOutputDir(originalFile, testGroupOutputDirForCompilation, stopFile)
val dceOutputDir = getOutputDir(originalFile, testGroupOutputDirForMinification, stopFile)
val perModuleOutputDir = getOutputDir(originalFile, testGroupOutputDirForPerModuleCompilation, stopFile)
val preModuleDceOutputDir = getOutputDir(originalFile, testGroupOutputDirForPerModuleMinification, stopFile)
val perModuleDceOutputDir = getOutputDir(originalFile, testGroupOutputDirForPerModuleMinification, stopFile)
val perFileOutputDir = getOutputDir(originalFile, testGroupOutputDirForPerFileCompilation, stopFile)
val perFileDceOutputDir = getOutputDir(originalFile, testGroupOutputDirForPerFileMinification, stopFile)
val minOutputDir = File(dceOutputDir, originalFile.nameWithoutExtension)
copy(JsEnvironmentConfigurator.getJsArtifactsOutputDir(testServices), outputDir)
copy(JsEnvironmentConfigurator.getJsArtifactsOutputDir(testServices, TranslationMode.FULL_PROD_MINIMIZED_NAMES), dceOutputDir)
copy(JsEnvironmentConfigurator.getJsArtifactsOutputDir(testServices, TranslationMode.PER_MODULE_DEV), perModuleOutputDir)
copy(JsEnvironmentConfigurator.getJsArtifactsOutputDir(testServices, TranslationMode.PER_MODULE_PROD_MINIMIZED_NAMES), preModuleDceOutputDir)
copy(JsEnvironmentConfigurator.getJsArtifactsOutputDir(testServices, TranslationMode.PER_MODULE_PROD_MINIMIZED_NAMES), perModuleDceOutputDir)
copy(JsEnvironmentConfigurator.getJsArtifactsOutputDir(testServices, TranslationMode.PER_FILE_DEV), perFileOutputDir)
copy(JsEnvironmentConfigurator.getJsArtifactsOutputDir(testServices, TranslationMode.PER_FILE_PROD_MINIMIZED_NAMES), perFileDceOutputDir)
copy(JsEnvironmentConfigurator.getMinificationJsArtifactsOutputDir(testServices), minOutputDir)
}
js/js.tests/test/org/jetbrains/kotlin/js/test/ir/AbstractJsPartialLinkageTestCase.kt
+2 -2
View File
@@ -23,11 +23,11 @@ import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.descriptors.DeclarationDescriptor
import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory
import org.jetbrains.kotlin.ir.backend.js.*
import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity
import org.jetbrains.kotlin.ir.backend.js.ic.CacheUpdater
import org.jetbrains.kotlin.ir.backend.js.ic.JsExecutableProducer
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.CompilationOutputs
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformer
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsGenerationGranularity
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.TranslationMode
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl
@@ -346,7 +346,7 @@ abstract class AbstractJsPartialLinkageTestCase(val compilerType: CompilerType)
relativeRequirePath = true
)
return jsExecutableProducer.buildExecutable(multiModule = true, outJsProgram = true).compilationOut
return jsExecutableProducer.buildExecutable(granularity = JsGenerationGranularity.PER_MODULE, outJsProgram = true).compilationOut
}
private val IrModuleFragment.exportName get() = "kotlin_${name.asStringStripSpecialMarkers()}"
js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt
+1 -1
View File
@@ -35,7 +35,7 @@ private class TestArtifactCache(val moduleName: String, val binaryAsts: MutableM
// TODO: It will be better to use saved fragments, but it doesn't work
// Merger.merge() + JsNode.resolveTemporaryNames() modify fragments,
// therefore the sequential calls produce different results
fragment = deserializeJsIrProgramFragment(it.value)
fragments = deserializeJsIrProgramFragment(it.value)
)
}
)
js/js.tests/test/org/jetbrains/kotlin/js/test/utils/RunnerUtils.kt
+8 -7
View File
@@ -50,7 +50,7 @@ private fun extractJsFiles(
fun copyInputJsFile(module: TestModule, inputJsFile: TestFile): String {
val newName = module.getNameFor(inputJsFile, testServices)
val targetFile = File(outputDir, newName)
targetFile.writeText(inputJsFile.originalContent)
targetFile.writeText(inputJsFile.originalContent.trim())
return targetFile.absolutePath
}
@@ -68,7 +68,7 @@ private fun extractJsFiles(
return before to after
}
fun getAdditionalFilePathes(testServices: TestServices, mode: TranslationMode = TranslationMode.FULL_DEV): List<String> {
fun getAdditionalFilePaths(testServices: TestServices, mode: TranslationMode = TranslationMode.FULL_DEV): List<String> {
return getAdditionalFiles(testServices, mode, true).map { it.absolutePath }
}
@@ -100,7 +100,7 @@ fun getAdditionalFiles(
return additionalFiles
}
fun getAdditionalMainFilePathes(testServices: TestServices, mode: TranslationMode = TranslationMode.FULL_DEV): List<String> {
fun getAdditionalMainFilePaths(testServices: TestServices, mode: TranslationMode = TranslationMode.FULL_DEV): List<String> {
return getAdditionalMainFiles(testServices, mode, shouldCopyFiles = true).map { it.absolutePath }
}
@@ -157,12 +157,13 @@ fun getAllFilesForRunner(
val outputFile = getModeOutputFilePath(testServices, module, mode)
val (inputJsFilesBefore, inputJsFilesAfter) = extractJsFiles(testServices, testServices.moduleStructure.modules, mode)
val additionalFiles = getAdditionalFilePathes(testServices, mode)
val additionalMainFiles = getAdditionalMainFilePathes(testServices, mode)
val additionalFiles = getAdditionalFilePaths(testServices, mode)
val additionalMainFiles = getAdditionalMainFilePaths(testServices, mode)
outputs.dependencies.forEach { (moduleId, _) ->
paths += outputFile.augmentWithModuleName(moduleId)
}
paths += outputFile
result[mode] = additionalFiles + inputJsFilesBefore + paths + commonFiles + additionalMainFiles + inputJsFilesAfter
@@ -171,8 +172,8 @@ fun getAllFilesForRunner(
return result
} else {
val (inputJsFilesBefore, inputJsFilesAfter) = extractJsFiles(testServices, testServices.moduleStructure.modules)
val additionalFiles = getAdditionalFilePathes(testServices)
val additionalMainFiles = getAdditionalMainFilePathes(testServices)
val additionalFiles = getAdditionalFilePaths(testServices)
val additionalMainFiles = getAdditionalMainFilePaths(testServices)
// Old BE
val outputDir = JsEnvironmentConfigurator.getJsArtifactsOutputDir(testServices)
val dceOutputDir = JsEnvironmentConfigurator.getJsArtifactsOutputDir(testServices, TranslationMode.FULL_PROD_MINIMIZED_NAMES)
js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/fir/FirJsBoxTestGenerated.java
Generated
+6
View File
@@ -2421,6 +2421,12 @@ public class FirJsBoxTestGenerated extends AbstractFirJsBoxTest {
runTest("js/js.translator/testData/box/esModules/jsExport/jsExportInClass.kt");
}
@Test
@TestMetadata("perFileExportedApi.kt")
public void testPerFileExportedApi() throws Exception {
runTest("js/js.translator/testData/box/esModules/jsExport/perFileExportedApi.kt");
}
@Test
@TestMetadata("recursiveExport.kt")
public void testRecursiveExport() throws Exception {
js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/ir/IrBoxJsES6TestGenerated.java
Generated
+6
View File
@@ -2527,6 +2527,12 @@ public class IrBoxJsES6TestGenerated extends AbstractIrBoxJsES6Test {
runTest("js/js.translator/testData/box/esModules/jsExport/jsExportInClass.kt");
}
@Test
@TestMetadata("perFileExportedApi.kt")
public void testPerFileExportedApi() throws Exception {
runTest("js/js.translator/testData/box/esModules/jsExport/perFileExportedApi.kt");
}
@Test
@TestMetadata("recursiveExport.kt")
public void testRecursiveExport() throws Exception {
js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/ir/IrBoxJsTestGenerated.java
Generated
+6
View File
@@ -2421,6 +2421,12 @@ public class IrBoxJsTestGenerated extends AbstractIrBoxJsTest {
runTest("js/js.translator/testData/box/esModules/jsExport/jsExportInClass.kt");
}
@Test
@TestMetadata("perFileExportedApi.kt")
public void testPerFileExportedApi() throws Exception {
runTest("js/js.translator/testData/box/esModules/jsExport/perFileExportedApi.kt");
}
@Test
@TestMetadata("recursiveExport.kt")
public void testRecursiveExport() throws Exception {