[JS IR] Generate debug info for JS injections

Also, fix the JS parser to collect the "correct" debug info (see the note in `translateJsCodeIntoStatementList`) #KT-53361 Fixed
2022-08-02 13:32:26 +02:00
parent 746c1b5903
commit d57ddc5f83
12 changed files with 182 additions and 47 deletions
@@ -23,7 +23,7 @@ import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity
 import org.jetbrains.kotlin.ir.backend.js.ir.JsIrBuilder
 import org.jetbrains.kotlin.ir.backend.js.lower.JsInnerClassesSupport
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsPolyfills
-import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.parseJsCode
+import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.translateJsCodeIntoStatementList
 import org.jetbrains.kotlin.ir.backend.js.utils.*
 import org.jetbrains.kotlin.ir.builders.declarations.addFunction
 import org.jetbrains.kotlin.ir.declarations.*
@@ -391,9 +391,10 @@ class JsIrBackendContext(
        val jsFunction = outlinedJsCodeFunctions[symbol]
        if (jsFunction != null) return jsFunction
        val jsFunAnnotation = symbol.owner.getAnnotation(JsAnnotations.jsFunFqn) ?: return null
-        val jsCode = jsFunAnnotation.getSingleConstStringArgument()
-        // FIXME: Provide debug info
-        val statements = parseJsCode(jsCode) ?: compilationException("Could not parse JS code", jsFunAnnotation)
+        val jsCode = jsFunAnnotation.getValueArgument(0)
+            ?: compilationException("@JsFun annotation must contain an argument", jsFunAnnotation)
+        val statements = translateJsCodeIntoStatementList(jsCode, this, symbol.owner)
+            ?: compilationException("Could not parse JS code", jsFunAnnotation)
        val parsedJsFunction = statements.singleOrNull()
            ?.safeAs<JsExpressionStatement>()
            ?.expression
@@ -134,7 +134,7 @@ private class JsCodeOutlineTransformer(
            return null

        val jsCodeArg = expression.getValueArgument(0) ?: compilationException("Expected js code string", expression)
-        val jsStatements = translateJsCodeIntoStatementList(jsCodeArg, backendContext) ?: return null
+        val jsStatements = translateJsCodeIntoStatementList(jsCodeArg, backendContext, container) ?: return null

        // Collect used Kotlin local variables and parameters.
        val scope = JsScopesCollector().apply { acceptList(jsStatements) }
@@ -69,7 +69,8 @@ class JsCallTransformer(private val jsOrJsFuncCall: IrCall, private val context:
            context.checkIfJsCode(jsOrJsFuncCall.symbol) -> {
                translateJsCodeIntoStatementList(
                    jsOrJsFuncCall.getValueArgument(0) ?: compilationException("JsCode is expected", jsOrJsFuncCall),
-                    context.staticContext.backendContext
+                    context.staticContext.backendContext,
+                    context.currentFile.fileEntry
                )
                    ?: compilationException("Cannot compute js code", jsOrJsFuncCall)
            }
@@ -5,9 +5,11 @@

 package org.jetbrains.kotlin.ir.backend.js.transformers.irToJs

-import org.jetbrains.kotlin.ir.backend.js.utils.getJsPolyfill
+import org.jetbrains.kotlin.ir.backend.js.utils.JsAnnotations
 import org.jetbrains.kotlin.ir.backend.js.utils.hasJsPolyfill
 import org.jetbrains.kotlin.ir.declarations.*
+import org.jetbrains.kotlin.ir.expressions.IrConst
+import org.jetbrains.kotlin.ir.util.getAnnotation
 import org.jetbrains.kotlin.js.backend.ast.JsStatement

 class JsPolyfills {
@@ -31,9 +33,11 @@ class JsPolyfills {
    private fun Iterable<IrDeclaration>.asImplementationList() =
        asSequence().asImplementationList()

-    private fun Sequence<IrDeclaration>.asImplementationList(): List<JsStatement> =
-        map { it.getJsPolyfill()!! }
-            .distinct()
-            .flatMap { parseJsCode(it).orEmpty() } // FIXME!!!
+    @Suppress("UNCHECKED_CAST")
+    private fun Sequence<IrDeclaration>.asImplementationList(): List<JsStatement> {
+        return map { it to it.getAnnotation(JsAnnotations.JsPolyfillFqn)!!.getValueArgument(0)!! }
+            .distinctBy { (it.second as IrConst<String>).value }
+            .flatMap { (container, polyfill) -> translateJsCodeIntoStatementList(polyfill, null, container).orEmpty() }
            .toList()
+    }
 }
@@ -7,10 +7,13 @@ package org.jetbrains.kotlin.ir.backend.js.transformers.irToJs

 import org.jetbrains.kotlin.backend.common.compilationException
 import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.IrFileEntry
 import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
+import org.jetbrains.kotlin.ir.backend.js.JsIrBackendContext
 import org.jetbrains.kotlin.ir.backend.js.JsStatementOrigins
 import org.jetbrains.kotlin.ir.backend.js.utils.*
 import org.jetbrains.kotlin.ir.declarations.IrClass
+import org.jetbrains.kotlin.ir.declarations.IrDeclaration
 import org.jetbrains.kotlin.ir.declarations.IrFunction
 import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
 import org.jetbrains.kotlin.ir.expressions.*
@@ -491,15 +494,22 @@ internal fun <T : JsNode> T.withSource(node: IrElement, context: JsGenerationCon
 private inline fun <T : JsNode> T.addSourceInfoIfNeed(node: IrElement, context: JsGenerationContext) {
    if (!context.staticContext.genSourcemaps) return

-    if (node.startOffset == UNDEFINED_OFFSET || node.endOffset == UNDEFINED_OFFSET) return
-
-    val fileEntry = context.currentFile.fileEntry
-
-    val path = fileEntry.name
-    val startLine = fileEntry.getLineNumber(node.startOffset)
-    val startColumn = fileEntry.getColumnNumber(node.startOffset)
+    val sourceInfo = node.getSourceInfo(context.currentFile.fileEntry) ?: return

    // TODO maybe it's better to fix in JsExpressionStatement
    val locationTarget = if (this is JsExpressionStatement) this.expression else this
-    locationTarget.source = JsLocation(path, startLine, startColumn)
+    locationTarget.source = sourceInfo
+}
+
+fun IrElement.getSourceInfo(container: IrDeclaration): JsLocation? {
+    val fileEntry = container.fileOrNull?.fileEntry ?: return null
+    return getSourceInfo(fileEntry)
+}
+
+fun IrElement.getSourceInfo(fileEntry: IrFileEntry): JsLocation? {
+    if (startOffset == UNDEFINED_OFFSET || endOffset == UNDEFINED_OFFSET) return null
+    val path = fileEntry.name
+    val startLine = fileEntry.getLineNumber(startOffset)
+    val startColumn = fileEntry.getColumnNumber(startOffset)
+    return JsLocation(path, startLine, startColumn)
 }
@@ -8,27 +8,44 @@ package org.jetbrains.kotlin.ir.backend.js.transformers.irToJs
 import com.google.gwt.dev.js.ThrowExceptionOnErrorReporter
 import com.google.gwt.dev.js.rhino.CodePosition
 import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.IrFileEntry
 import org.jetbrains.kotlin.ir.backend.js.JsIrBackendContext
 import org.jetbrains.kotlin.ir.backend.js.lower.PropertyLazyInitLowering
+import org.jetbrains.kotlin.ir.declarations.IrDeclaration
 import org.jetbrains.kotlin.ir.declarations.IrVariable
 import org.jetbrains.kotlin.ir.expressions.*
+import org.jetbrains.kotlin.ir.util.fileOrNull
 import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
 import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
 import org.jetbrains.kotlin.ir.visitors.acceptVoid
-import org.jetbrains.kotlin.js.backend.ast.JsFunctionScope
-import org.jetbrains.kotlin.js.backend.ast.JsProgram
-import org.jetbrains.kotlin.js.backend.ast.JsRootScope
-import org.jetbrains.kotlin.js.backend.ast.JsStatement
+import org.jetbrains.kotlin.js.backend.ast.*
 import org.jetbrains.kotlin.js.parser.parseExpressionOrStatement

-// Returns null if constant expression could not be parsed
-fun translateJsCodeIntoStatementList(code: IrExpression, context: JsIrBackendContext): List<JsStatement>? {
-    // TODO: support proper symbol linkage and label clash resolution
-    // FIXME: Provide debug info
-    return parseJsCode(foldString(code, context) ?: return null)
-}
+/**
+ * Returns null if constant expression could not be parsed.
+ */
+fun translateJsCodeIntoStatementList(code: IrExpression, context: JsIrBackendContext?, container: IrDeclaration) =
+    translateJsCodeIntoStatementList(
+        code,
+        context,
+        code.getSourceInfo(container) ?: container.fileOrNull?.fileEntry?.let { JsLocation(it.name, 0, 0) }
+    )
+
+/**
+ * Returns null if constant expression could not be parsed.
+ */
+fun translateJsCodeIntoStatementList(code: IrExpression, context: JsIrBackendContext?, fileEntry: IrFileEntry) =
+    translateJsCodeIntoStatementList(code, context, code.getSourceInfo(fileEntry) ?: JsLocation(fileEntry.name, 0, 0))
+
+private fun translateJsCodeIntoStatementList(
+    code: IrExpression,
+    context: JsIrBackendContext?,
+    sourceInfo: JsLocation?
+): List<JsStatement>? {
+    // TODO: support proper symbol linkage and label clash resolution
+    val (fileName, startLine, offset) = sourceInfo ?: JsLocation("<js-code>", 0, 0)
+    val jsCode = foldString(code, context) ?: return null

-fun parseJsCode(jsCode: String): List<JsStatement>? {
    // Parser can change local or global scope.
    // In case of js we want to keep new local names,
    // but no new global ones.
@@ -36,12 +53,17 @@ fun parseJsCode(jsCode: String): List<JsStatement>? {
    val temporaryRootScope = JsRootScope(JsProgram())
    val currentScope = JsFunctionScope(temporaryRootScope, "js")

-    // TODO: write debug info, see how it's done in CallExpressionTranslator.parseJsCode
-
-    return parseExpressionOrStatement(jsCode, ThrowExceptionOnErrorReporter, currentScope, CodePosition(0, 0), "<js-code>")
+    // NOTE: emitting the correct debug info for JS injections is a non-trivial task, because the injection may consist of
+    // constant-evaluated strings, whose origin is hard to track in the JS parser. Also, the presence of explicit \n characters in
+    // the JS string literal breaks the debug info, because at this level we are unable to distinguish multiline string literals and
+    // single-line string literals with explicit \n in it.
+    //
+    // So we try to generate the debug info on the best-effort basis. It should work correctly with plain string literals without
+    // concatenations, interpolations or backslash replacements like \n.
+    return parseExpressionOrStatement(jsCode, ThrowExceptionOnErrorReporter, currentScope, CodePosition(startLine, offset), fileName)
 }

-fun foldString(expression: IrExpression, context: JsIrBackendContext): String? {
+fun foldString(expression: IrExpression, context: JsIrBackendContext?): String? {
    val builder = StringBuilder()
    var foldingFailed = false
    expression.acceptVoid(object : IrElementVisitorVoid {
@@ -65,7 +87,7 @@ fun foldString(expression: IrExpression, context: JsIrBackendContext): String? {
        override fun visitGetField(expression: IrGetField) {
            val owner = expression.symbol.owner
            owner.initializer?.expression?.acceptVoid(this)
-                ?: context.fieldToInitializer[owner]?.acceptVoid(this)
+                ?: context?.fieldToInitializer?.get(owner)?.acceptVoid(this)
        }

        override fun visitCall(expression: IrCall) {
@@ -101,12 +123,12 @@ fun foldString(expression: IrExpression, context: JsIrBackendContext): String? {
    return builder.toString()
 }

-private class InitFunVisitor(private val context: JsIrBackendContext) : IrElementVisitorVoid {
+private class InitFunVisitor(private val context: JsIrBackendContext?) : IrElementVisitorVoid {
    override fun visitElement(element: IrElement) {
        element.acceptChildrenVoid(this)
    }

    override fun visitSetField(expression: IrSetField) {
-        context.fieldToInitializer[expression.symbol.owner] = expression.value
+        context?.fieldToInitializer?.set(expression.symbol.owner, expression.value)
    }
 }
@@ -51,9 +51,6 @@ fun IrAnnotationContainer.getJsQualifier(): String? =
 fun IrAnnotationContainer.getJsName(): String? =
    getAnnotation(JsAnnotations.jsNameFqn)?.getSingleConstStringArgument()

-fun IrAnnotationContainer.getJsPolyfill(): String? =
-    getAnnotation(JsAnnotations.JsPolyfillFqn)?.getSingleConstStringArgument()
-
 fun IrAnnotationContainer.hasJsPolyfill(): Boolean =
    hasAnnotation(JsAnnotations.JsPolyfillFqn)

@@ -0,0 +1,80 @@
+// TARGET_BACKEND: JS_IR
+
+// FILE: a.kt
+@JsExport
+fun exclamate(a: String) =
+    "$a!"
+
+@Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE")
+@JsFun("""
+    function (a, b) {
+        return _.exclamate(a) +
+            _.exclamate(b);
+    }
+""")
+external fun exclamateAndConcat(a: String, b: String): String
+
+@Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE")
+@JsPolyfill("""
+if (typeof String.prototype.myAwesomePolyfill === "undefined") {
+    Object.defineProperty(String.prototype, "myAwesomePolyfill", {
+        value: function () {
+            return this + "!";
+        }
+    });
+}
+""")
+internal inline fun String.myAwesomePolyfill(): Boolean =
+    asDynamic()
+        .myAwesomePolyfill()
+
+val walter1VarDecl = "var walter1 = _.exclamate('Walter');"
+val jesse1VarDecl = "var jesse1 = _.exclamate(jesse);"
+
+// FILE: test.kt
+
+fun noop() {}
+
+fun box() {
+    noop()
+    exclamateAndConcat(
+        "hello",
+        "world")
+    val jesse = "Jesse"
+    js(
+        "_.exclamate(jesse);") // Local variable is captured
+    js(
+        "_.exclamate('Walter');") // No local variables captured
+    js(
+        walter1VarDecl +
+        "_.exclamate(walter1);")
+    js(
+        jesse1VarDecl +
+                "_.exclamate(jesse1);")
+    
+    "foo".myAwesomePolyfill()
+}
+
+// EXPECTATIONS
+// test.kt:39 box
+// test.kt:41 box
+// test.kt:42 box
+// a.kt:11 box
+// a.kt:6 exclamate
+// a.kt:12 box
+// a.kt:6 exclamate
+// test.kt:43 box
+// test.kt:45 box
+// a.kt:6 exclamate
+// test.kt:47 box
+// a.kt:6 exclamate
+// test.kt:49 box
+// a.kt:6 exclamate
+// test.kt:49 box
+// a.kt:6 exclamate
+// test.kt:52 box
+// a.kt:6 exclamate
+// test.kt:52 box
+// a.kt:6 exclamate
+// a.kt:29 box
+// a.kt:22 value
@@ -67,6 +67,7 @@ final class LineBuffer {
    LineBuffer(Reader in, CodePosition position) {
        this.in = in;
        this.lineno = position.getLine();
+        this.columnno = position.getOffset();
        this.lineStart = -position.getOffset();
    }

@@ -77,14 +78,16 @@ final class LineBuffer {

            int c = buffer[offset];
            ++offset;
+            prevColumnno = columnno++;

            if ((c & EOL_HINT_MASK) == 0) {
                switch (c) {
                    case '\r':
                        // if the next character is a newline, skip past it.
                        if (offset != end) {
-                            if (buffer[offset] == '\n')
+                            if (buffer[offset] == '\n') {
                                ++offset;
+                            }
                        } else {
                            // set a flag for fill(), in case the first char
                            // of the next fill is a newline.
@@ -95,6 +98,8 @@ final class LineBuffer {
                        prevStart = lineStart;
                        lineStart = offset;
                        lineno++;
+                        prevColumnno = columnno;
+                        columnno = 0;
                        return '\n';
                }
            }
@@ -120,10 +125,12 @@ final class LineBuffer {
        if (offset == 0) // Same as if (hitEOF)
            return;
        offset--;
+        columnno--;
        int c = buffer[offset];
        if ((c & EOL_HINT_MASK) == 0 && isEndOfLine(c)) {
            lineStart = prevStart;
            lineno--;
+            columnno = prevColumnno;
        }
    }

@@ -147,6 +154,7 @@ final class LineBuffer {
        }

        ++offset;
+        prevColumnno = columnno++;
    }

    int peek() throws IOException {
@@ -182,6 +190,7 @@ final class LineBuffer {
            int c = buffer[offset];
            if (test == c) {
                ++offset;
+                prevColumnno = columnno++;
                return true;
            }
            if (c < 128 || !formatChar(c)) {
@@ -292,7 +301,8 @@ final class LineBuffer {
        // likely get to EOF for real) by doing yet another fill().
        if (lastWasCR) {
            if (buffer[0] == '\n') {
-              offset++;
+              ++offset;
+              prevColumnno = columnno++;
              if (end == 1)
                  return fill();
            }
@@ -303,6 +313,9 @@ final class LineBuffer {
    }

    int getLineno() { return lineno; }
+
+    int getColumnno() { return columnno; };
+
    boolean eof() { return hitEOF; }

    private static boolean formatChar(int c) {
@@ -322,6 +335,8 @@ final class LineBuffer {
    private int end = 0;
    private int otherEnd;
    private int lineno;
+    private int columnno;
+    private int prevColumnno = 0;

    private int lineStart = 0;
    private int otherStart = 0;
@@ -503,6 +503,7 @@ public class TokenStream {
        flags = 0;
        secondToLastPosition = position;
        lastPosition = position;
+        tokenPosition = position;
        lastTokenPosition = position;
    }

@@ -624,7 +625,7 @@ public class TokenStream {
            }
        } while (isJSSpace(c) || c == '\n');

-        tokenPosition = new CodePosition(in.getLineno(), Math.max(in.getOffset() - 1, 0));
+        tokenPosition = new CodePosition(in.getLineno(), in.getColumnno() - 1);
        if (c == EOF_CHAR)
            return EOF;
        if (c != '-' && c != '\n')
@@ -247,6 +247,12 @@ public class IrJsSteppingTestGenerated extends AbstractIrJsSteppingTest {
        runTest("compiler/testData/debug/stepping/inlineSimpleCall.kt");
    }

+    @Test
+    @TestMetadata("jsCode.kt")
+    public void testJsCode() throws Exception {
+        runTest("compiler/testData/debug/stepping/jsCode.kt");
+    }
+
    @Test
    @TestMetadata("kt15259.kt")
    public void testKt15259() throws Exception {
@@ -5,7 +5,5 @@ fun foo() {
    println("after: $x")
 }

-// LINES(JS):    1 6 2 2 3 3 4 4     5 5
-// LINES(JS_IR):     2 2 3 3     2 2 5 5
-// FIXME:                    ^^^^^^^
-//                           js function call debug info is incorrect
+// LINES(JS):    1 6 2 2 3 3 4 4 5 5
+// LINES(JS_IR):     2 2 3 3 4 4 5 5