[Wasm] Add a basic end-to-end sourcemap generation in wasm backend

More constructions/instructions will be supported separately.
This commit is contained in:
Zalim Bashorov
2022-12-06 16:19:59 +01:00
committed by teamcity
parent 1c4614e93b
commit 49c3ba33f1
14 changed files with 250 additions and 42 deletions
@@ -367,18 +367,23 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
if (arguments.irDce) {
eliminateDeadDeclarations(allModules, backendContext)
}
val sourceMapFileName = if (configuration.getBoolean(JSConfigurationKeys.SOURCE_MAP)) "$outputName.map" else null
val res = compileWasm(
allModules = allModules,
backendContext = backendContext,
emitNameSection = arguments.wasmDebug,
allowIncompleteImplementations = arguments.irDce,
generateWat = true,
sourceMapFileName = sourceMapFileName
)
writeCompilationResult(
result = res,
dir = outputDir,
fileNameBase = outputName
fileNameBase = outputName,
sourceMapFileName = sourceMapFileName
)
return OK
@@ -11,20 +11,31 @@ import org.jetbrains.kotlin.backend.common.serialization.linkerissues.checkNoUnb
import org.jetbrains.kotlin.backend.wasm.ir2wasm.WasmCompiledModuleFragment
import org.jetbrains.kotlin.backend.wasm.ir2wasm.WasmModuleFragmentGenerator
import org.jetbrains.kotlin.backend.wasm.lower.markExportedDeclarations
import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.ir.backend.js.MainModule
import org.jetbrains.kotlin.ir.backend.js.ModulesStructure
import org.jetbrains.kotlin.ir.backend.js.SourceMapsInfo
import org.jetbrains.kotlin.ir.backend.js.loadIr
import org.jetbrains.kotlin.ir.declarations.IrFactory
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.ir.util.ExternalDependenciesGenerator
import org.jetbrains.kotlin.ir.util.patchDeclarationParents
import org.jetbrains.kotlin.js.sourceMap.SourceFilePathResolver
import org.jetbrains.kotlin.js.sourceMap.SourceMap3Builder
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.wasm.ir.source.location.SourceLocation
import org.jetbrains.kotlin.wasm.ir.convertors.WasmIrToBinary
import org.jetbrains.kotlin.wasm.ir.convertors.WasmIrToText
import org.jetbrains.kotlin.wasm.ir.source.location.SourceLocationMapping
import java.io.ByteArrayOutputStream
import java.io.File
class WasmCompilerResult(val wat: String?, val js: String, val wasm: ByteArray)
class WasmCompilerResult(
val wat: String?,
val js: String,
val wasm: ByteArray,
val sourceMap: String?
)
fun compileToLoweredIr(
depsDescriptors: ModulesStructure,
@@ -77,6 +88,7 @@ fun compileWasm(
emitNameSection: Boolean = false,
allowIncompleteImplementations: Boolean = false,
generateWat: Boolean = false,
sourceMapFileName: String? = null,
): WasmCompilerResult {
val compiledWasmModule = WasmCompiledModuleFragment(backendContext.irBuiltIns)
val codeGenerator = WasmModuleFragmentGenerator(backendContext, compiledWasmModule, allowIncompleteImplementations = allowIncompleteImplementations)
@@ -95,16 +107,65 @@ fun compileWasm(
val js = compiledWasmModule.generateJs()
val os = ByteArrayOutputStream()
WasmIrToBinary(os, linkedModule, allModules.last().descriptor.name.asString(), emitNameSection).appendWasmModule()
val sourceLocationMappings =
if (sourceMapFileName != null) mutableListOf<SourceLocationMapping>() else null
val wasmIrToBinary =
WasmIrToBinary(
os,
linkedModule,
allModules.last().descriptor.name.asString(),
emitNameSection,
sourceMapFileName,
sourceLocationMappings
)
wasmIrToBinary.appendWasmModule()
val byteArray = os.toByteArray()
return WasmCompilerResult(
wat = wat,
js = js,
wasm = byteArray
wasm = byteArray,
sourceMap = generateSourceMap(backendContext.configuration, sourceLocationMappings)
)
}
private fun generateSourceMap(
configuration: CompilerConfiguration,
sourceLocationMappings: MutableList<SourceLocationMapping>?
): String? {
if (sourceLocationMappings == null) return null
val sourceMapsInfo = SourceMapsInfo.from(configuration) ?: return null
val sourceMapBuilder =
SourceMap3Builder(null, { error("This should not be called for Kotlin/Wasm") }, sourceMapsInfo.sourceMapPrefix)
val pathResolver =
SourceFilePathResolver.create(sourceMapsInfo.sourceRoots, sourceMapsInfo.sourceMapPrefix, sourceMapsInfo.outputDir)
var prev: SourceLocation? = null
for (mapping in sourceLocationMappings) {
val location = mapping.sourceLocation as? SourceLocation.Location ?: continue
if (location == prev) continue
prev = location
location.apply {
// TODO resulting path goes too deep since temporary directory we compiled first is deeper than final destination.
val relativePath = pathResolver.getPathRelativeToSourceRoots(File(file)).substring(3)
sourceMapBuilder.addMapping(relativePath, null, { null }, line, column, null, mapping.offset)
}
}
return sourceMapBuilder.build()
}
fun WasmCompiledModuleFragment.generateJs(): String {
//language=js
val runtime = """
@@ -170,7 +231,8 @@ fun generateJsWasmLoader(wasmFilePath: String, externalJs: String): String =
fun writeCompilationResult(
result: WasmCompilerResult,
dir: File,
fileNameBase: String = "index",
fileNameBase: String,
sourceMapFileName: String?
) {
dir.mkdirs()
if (result.wat != null) {
@@ -180,4 +242,8 @@ fun writeCompilationResult(
val jsWithLoader = generateJsWasmLoader("./$fileNameBase.wasm", result.js)
File(dir, "$fileNameBase.mjs").writeText(jsWithLoader)
if (sourceMapFileName != null) {
File(dir, sourceMapFileName).writeText(result.sourceMap!!)
}
}
@@ -27,6 +27,8 @@ import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
import org.jetbrains.kotlin.ir.visitors.acceptVoid
import org.jetbrains.kotlin.js.config.JSConfigurationKeys
import org.jetbrains.kotlin.wasm.ir.*
import org.jetbrains.kotlin.wasm.ir.source.location.SourceLocation
import org.jetbrains.kotlin.wasm.ir.source.location.withLocation
class BodyGenerator(
val context: WasmFunctionCodegenContext,
@@ -420,7 +422,9 @@ class BodyGenerator(
} else {
// Static function call
body.buildCall(context.referenceFunction(function.symbol))
withLocation(call.getSourceLocation()) {
body.buildCall(context.referenceFunction(function.symbol), location)
}
}
// Unit types don't cross function boundaries
@@ -646,7 +650,7 @@ class BodyGenerator(
body.buildGetLocal(context.referenceLocal(0))
}
body.buildInstr(WasmOp.RETURN)
body.buildInstr(WasmOp.RETURN, expression.getSourceLocation())
}
internal fun generateWithExpectedType(expression: IrExpression, expectedType: IrType) {
@@ -883,4 +887,16 @@ class BodyGenerator(
return false
}
private fun IrExpression.getSourceLocation(): SourceLocation {
val fileEntry = context.irFunction.fileEntry
val path = fileEntry.name
val startLine = fileEntry.getLineNumber(startOffset)
val startColumn = fileEntry.getColumnNumber(startOffset)
if (startLine < 0 || startColumn < 0) return SourceLocation.NoLocation
return SourceLocation.Location(path, startLine, startColumn)
}
}
@@ -214,7 +214,7 @@ abstract class BasicWasmBoxTest(
)
}
writeCompilationResult(res, dir)
writeCompilationResult(res, dir, "index", sourceMapFileName = null)
File(dir, "test.mjs").writeText(testJs)
ExternalTool(System.getProperty("javascript.engine.path.V8"))
.run(
+1
View File
@@ -7,6 +7,7 @@ plugins {
dependencies {
implementation(kotlinStdlib())
implementation(kotlinxCollectionsImmutable())
testImplementation(commonDependency("junit:junit"))
testCompileOnly(project(":kotlin-test:kotlin-test-jvm"))
testCompileOnly(project(":kotlin-test:kotlin-test-junit"))
@@ -5,6 +5,8 @@
package org.jetbrains.kotlin.wasm.ir
import org.jetbrains.kotlin.wasm.ir.source.location.SourceLocation
class WasmModule(
val functionTypes: List<WasmFunctionType> = emptyList(),
@@ -169,10 +171,30 @@ class WasmStructFieldDeclaration(
val isMutable: Boolean
)
class WasmInstr(
sealed class WasmInstr(
val operator: WasmOp,
val immediates: List<WasmImmediate> = emptyList()
)
) {
abstract val location: SourceLocation?
}
class WasmInstrWithLocation(
operator: WasmOp,
immediates: List<WasmImmediate>,
override val location: SourceLocation
) : WasmInstr(operator, immediates) {
constructor(
operator: WasmOp,
location: SourceLocation
) : this(operator, emptyList(), location)
}
class WasmInstrWithoutLocation(
operator: WasmOp,
immediates: List<WasmImmediate> = emptyList(),
) : WasmInstr(operator, immediates) {
override val location: SourceLocation? get() = null
}
data class WasmLimits(
val minSize: UInt,
@@ -182,4 +204,4 @@ data class WasmLimits(
data class WasmImportPair(
val moduleName: String,
val declarationName: String
)
)
@@ -5,12 +5,19 @@
package org.jetbrains.kotlin.wasm.ir
import org.jetbrains.kotlin.wasm.ir.source.location.SourceLocation
abstract class WasmExpressionBuilder {
abstract fun buildInstr(op: WasmOp, vararg immediates: WasmImmediate)
abstract fun buildInstr(op: WasmOp, location: SourceLocation, vararg immediates: WasmImmediate)
fun buildInstr(op: WasmOp, vararg immediates: WasmImmediate) {
buildInstr(op, SourceLocation.TBDLocation, *immediates)
}
abstract var numberOfNestedBlocks: Int
fun buildConstI32(value: Int) {
buildInstr(WasmOp.I32_CONST, WasmImmediate.ConstI32(value))
fun buildConstI32(value: Int, location: SourceLocation = SourceLocation.TBDLocation) {
buildInstr(WasmOp.I32_CONST, location, WasmImmediate.ConstI32(value))
}
fun buildConstI64(value: Long) {
@@ -105,8 +112,8 @@ abstract class WasmExpressionBuilder {
buildBrInstr(WasmOp.BR_IF, absoluteBlockLevel)
}
fun buildCall(symbol: WasmSymbol<WasmFunction>) {
buildInstr(WasmOp.CALL, WasmImmediate.FuncIdx(symbol))
fun buildCall(symbol: WasmSymbol<WasmFunction>, location: SourceLocation = SourceLocation.TBDLocation) {
buildInstr(WasmOp.CALL, location, WasmImmediate.FuncIdx(symbol))
}
fun buildCallIndirect(
@@ -5,6 +5,8 @@
package org.jetbrains.kotlin.wasm.ir
import org.jetbrains.kotlin.wasm.ir.source.location.SourceLocation
private fun WasmOp.isOutCfgNode() = when (this) {
WasmOp.UNREACHABLE, WasmOp.RETURN, WasmOp.THROW, WasmOp.RETHROW, WasmOp.BR, WasmOp.BR_TABLE -> true
else -> false
@@ -28,8 +30,8 @@ class WasmIrExpressionBuilder(
get() = expression.lastOrNull()
private var eatEverythingUntilLevel: Int? = null
private fun addInstruction(op: WasmOp, immediates: Array<out WasmImmediate>) {
val newInstruction = WasmInstr(op, immediates.toList())
private fun addInstruction(op: WasmOp, location: SourceLocation, immediates: Array<out WasmImmediate>) {
val newInstruction = WasmInstrWithLocation(op, immediates.toList(), location)
expression.add(newInstruction)
}
@@ -46,21 +48,21 @@ class WasmIrExpressionBuilder(
return eatLevel
}
override fun buildInstr(op: WasmOp, vararg immediates: WasmImmediate) {
override fun buildInstr(op: WasmOp, location: SourceLocation, vararg immediates: WasmImmediate) {
val currentEatUntil = getCurrentEatLevel(op)
if (currentEatUntil != null) {
if (currentEatUntil <= numberOfNestedBlocks) return
} else {
if (op.isOutCfgNode()) {
eatEverythingUntilLevel = numberOfNestedBlocks
addInstruction(op, immediates)
addInstruction(op, location, immediates)
return
}
}
val lastInstruction = lastInstr
if (lastInstruction == null) {
addInstruction(op, immediates)
addInstruction(op, location, immediates)
return
}
val lastOperator = lastInstruction.operator
@@ -78,13 +80,13 @@ class WasmIrExpressionBuilder(
val localGetNumber = (immediates.firstOrNull() as? WasmImmediate.LocalIdx)?.value
if (localGetNumber == localSetNumber) {
expression.removeLast()
addInstruction(WasmOp.LOCAL_TEE, immediates)
addInstruction(WasmOp.LOCAL_TEE, location, immediates)
return
}
}
}
addInstruction(op, immediates)
addInstruction(op, location, immediates)
}
override var numberOfNestedBlocks: Int = 0
@@ -98,4 +100,4 @@ inline fun buildWasmExpression(body: WasmExpressionBuilder.() -> Unit): MutableL
val res = mutableListOf<WasmInstr>()
WasmIrExpressionBuilder(res).body()
return res
}
}
@@ -442,8 +442,8 @@ class WasmBinaryToIR(val b: MyByteReader) {
}
}
return WasmInstr(op, immediates)
// We don't need location in Binary -> WasmIR, yet.
return WasmInstrWithoutLocation(op, immediates)
}
private fun readTypeDeclaration(): WasmTypeDeclaration {
@@ -3,16 +3,30 @@
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
@file:OptIn(ExperimentalUnsignedTypes::class)
package org.jetbrains.kotlin.wasm.ir.convertors
import org.jetbrains.kotlin.wasm.ir.*
import java.io.ByteArrayOutputStream
import java.io.OutputStream
import kotlinx.collections.immutable.*
import org.jetbrains.kotlin.wasm.ir.source.location.Box
import org.jetbrains.kotlin.wasm.ir.source.location.SourceLocation
import org.jetbrains.kotlin.wasm.ir.source.location.SourceLocationMapping
class WasmIrToBinary(outputStream: OutputStream, val module: WasmModule, val moduleName: String, val emitNameSection: Boolean) {
var b: ByteWriter = ByteWriter.OutputStream(outputStream)
class WasmIrToBinary(
outputStream: OutputStream,
val module: WasmModule,
val moduleName: String,
val emitNameSection: Boolean,
private val sourceMapFileName: String? = null,
private val sourceLocationMappings: MutableList<SourceLocationMapping>? = null
) {
private var b: ByteWriter = ByteWriter.OutputStream(outputStream)
// "Stack" of offsets waiting initialization.
// Since blocks has as a prefix variable length number encoding its size we can't calculate absolute offsets inside those blocks
// until we generate whole block and generate size. So, we put them into "stack" and initialize as soo as we have all required data.
private var offsets = persistentListOf<Box>()
fun appendWasmModule() {
b.writeUInt32(0x6d736100u) // WebAssembly magic
@@ -114,10 +128,18 @@ class WasmIrToBinary(outputStream: OutputStream, val module: WasmModule, val mod
data.forEach { appendData(it) }
}
//text section (should be placed after data)
// text section (should be placed after data)
if (emitNameSection) {
appendTextSection(definedFunctions)
}
if (sourceMapFileName != null) {
// Custom section with URL to sourcemap
appendSection(0u) {
b.writeString("sourceMappingURL")
b.writeString(sourceMapFileName)
}
}
}
}
@@ -183,6 +205,10 @@ class WasmIrToBinary(outputStream: OutputStream, val module: WasmModule, val mod
}
private fun appendInstr(instr: WasmInstr) {
instr.location?.let {
sourceLocationMappings?.add(SourceLocationMapping(offsets + Box(b.written), it))
}
val opcode = instr.operator.opcode
if (opcode > 0xFF) {
b.writeByte((opcode ushr 8).toByte())
@@ -241,14 +267,21 @@ class WasmIrToBinary(outputStream: OutputStream, val module: WasmModule, val mod
withVarUInt32PayloadSizePrepended { content() }
}
@OptIn(ExperimentalStdlibApi::class)
fun withVarUInt32PayloadSizePrepended(fn: () -> Unit) {
private fun withVarUInt32PayloadSizePrepended(fn: () -> Unit) {
val box = Box(-1)
val previousOffsets = offsets
offsets += box
val previousWriter = b
val newWriter = b.createTemp()
b = newWriter
fn()
b = previousWriter
b.writeVarUInt32(newWriter.written)
box.value = b.written
offsets = previousOffsets
b.write(newWriter)
}
@@ -350,7 +383,7 @@ class WasmIrToBinary(outputStream: OutputStream, val module: WasmModule, val mod
}
appendType(c.type)
b.writeVarUInt1(c.isMutable)
appendExpr(c.init)
appendExpr(c.init, SourceLocation.TBDLocation)
}
private fun appendTag(t: WasmTag) {
@@ -365,9 +398,9 @@ class WasmIrToBinary(outputStream: OutputStream, val module: WasmModule, val mod
b.writeVarUInt32(t.type.id!!)
}
private fun appendExpr(expr: Iterable<WasmInstr>) {
private fun appendExpr(expr: Iterable<WasmInstr>, location: SourceLocation) {
expr.forEach { appendInstr(it) }
appendInstr(WasmInstr(WasmOp.END))
appendInstr(WasmInstrWithLocation(WasmOp.END, location))
}
private fun appendExport(export: WasmExport<*>) {
@@ -393,7 +426,7 @@ class WasmIrToBinary(outputStream: OutputStream, val module: WasmModule, val mod
funcIndices.forEach { b.writeVarUInt32(it) }
} else {
element.values.forEach {
appendExpr((it as WasmTable.Value.Expression).expr)
appendExpr((it as WasmTable.Value.Expression).expr, SourceLocation.TBDLocation)
}
}
}
@@ -417,18 +450,18 @@ class WasmIrToBinary(outputStream: OutputStream, val module: WasmModule, val mod
when {
tableId == 0 && isFuncIndices -> {
b.writeByte(0x0)
appendExpr(mode.offset)
appendExpr(mode.offset, SourceLocation.TBDLocation)
}
isFuncIndices -> {
b.writeByte(0x2)
appendModuleFieldReference(mode.table)
appendExpr(mode.offset)
appendExpr(mode.offset, SourceLocation.TBDLocation)
writeTypeOrKind()
}
else -> {
b.writeByte(0x6)
appendModuleFieldReference(mode.table)
appendExpr(mode.offset)
appendExpr(mode.offset, SourceLocation.TBDLocation)
writeTypeOrKind()
}
}
@@ -452,7 +485,7 @@ class WasmIrToBinary(outputStream: OutputStream, val module: WasmModule, val mod
}
}
appendExpr(function.instructions)
appendExpr(function.instructions, SourceLocation.TBDLocation)
}
}
@@ -465,7 +498,7 @@ class WasmIrToBinary(outputStream: OutputStream, val module: WasmModule, val mod
b.writeByte(2)
b.writeVarUInt32(mode.memoryIdx)
}
appendExpr(mode.offset)
appendExpr(mode.offset, SourceLocation.TBDLocation)
}
WasmDataMode.Passive -> b.writeByte(1)
}
@@ -0,0 +1,8 @@
/*
* Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.wasm.ir.source.location
class Box(var value: Int)
@@ -0,0 +1,15 @@
/*
* Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.wasm.ir.source.location
@JvmInline
value class LocationHolder(val location: SourceLocation)
inline fun <R> withLocation(location: SourceLocation, body: LocationHolder.() -> R): R = LocationHolder(location).body()
inline fun <R> withNoLocation(body: LocationHolder.() -> R): R = withLocation(SourceLocation.NoLocation, body)
inline fun <R> withTBDLocation(body: LocationHolder.() -> R): R = withLocation<R>(SourceLocation.TBDLocation, body)
@@ -0,0 +1,13 @@
/*
* Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.wasm.ir.source.location
sealed class SourceLocation {
object NoLocation: SourceLocation()
object TBDLocation: SourceLocation()
data class Location(val file: String, val line: Int, val column: Int) : SourceLocation()
}
@@ -0,0 +1,20 @@
/*
* Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.wasm.ir.source.location
class SourceLocationMapping(
// Offsets in generating binary, initialized lazily. Since blocks has as a prefix variable length number encoding its size
// we can't calculate absolute offsets inside those blocks until we generate whole block and generate size.
private val offsets: List<Box>,
val sourceLocation: SourceLocation
) {
val offset by lazy {
offsets.sumOf {
assert(it.value >= 0) { "Offset must be >=0 but ${it.value}" }
it.value
}
}
}