[JS IR] Generate debug info for JS injections

Also, fix the JS parser to collect the "correct" debug info
(see the note in `translateJsCodeIntoStatementList`)

#KT-53361 Fixed

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/JsIrBackendContext.kt

@@ -23,7 +23,7 @@ import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity
 import org.jetbrains.kotlin.ir.backend.js.ir.JsIrBuilder
 import org.jetbrains.kotlin.ir.backend.js.lower.JsInnerClassesSupport
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsPolyfills
-import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.parseJsCode
+import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.translateJsCodeIntoStatementList
 import org.jetbrains.kotlin.ir.backend.js.utils.*
 import org.jetbrains.kotlin.ir.builders.declarations.addFunction
 import org.jetbrains.kotlin.ir.declarations.*
@@ -391,9 +391,10 @@ class JsIrBackendContext(
         val jsFunction = outlinedJsCodeFunctions[symbol]
         if (jsFunction != null) return jsFunction
         val jsFunAnnotation = symbol.owner.getAnnotation(JsAnnotations.jsFunFqn) ?: return null
-        val jsCode = jsFunAnnotation.getSingleConstStringArgument()
+        val jsCode = jsFunAnnotation.getValueArgument(0)
-        // FIXME: Provide debug info
+            ?: compilationException("@JsFun annotation must contain an argument", jsFunAnnotation)
-        val statements = parseJsCode(jsCode) ?: compilationException("Could not parse JS code", jsFunAnnotation)
+        val statements = translateJsCodeIntoStatementList(jsCode, this, symbol.owner)
+            ?: compilationException("Could not parse JS code", jsFunAnnotation)
         val parsedJsFunction = statements.singleOrNull()
             ?.safeAs<JsExpressionStatement>()
             ?.expression

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/lower/JsCodeOutliningLowering.kt

@@ -134,7 +134,7 @@ private class JsCodeOutlineTransformer(
             return null
 
         val jsCodeArg = expression.getValueArgument(0) ?: compilationException("Expected js code string", expression)
-        val jsStatements = translateJsCodeIntoStatementList(jsCodeArg, backendContext) ?: return null
+        val jsStatements = translateJsCodeIntoStatementList(jsCodeArg, backendContext, container) ?: return null
 
         // Collect used Kotlin local variables and parameters.
         val scope = JsScopesCollector().apply { acceptList(jsStatements) }

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/transformers/irToJs/JsCallTransformer.kt

@@ -69,7 +69,8 @@ class JsCallTransformer(private val jsOrJsFuncCall: IrCall, private val context:
             context.checkIfJsCode(jsOrJsFuncCall.symbol) -> {
                 translateJsCodeIntoStatementList(
                     jsOrJsFuncCall.getValueArgument(0) ?: compilationException("JsCode is expected", jsOrJsFuncCall),
-                    context.staticContext.backendContext
+                    context.staticContext.backendContext,
+                    context.currentFile.fileEntry
                 )
                     ?: compilationException("Cannot compute js code", jsOrJsFuncCall)
             }

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/transformers/irToJs/JsPolyfills.kt

@@ -5,9 +5,11 @@
 
 package org.jetbrains.kotlin.ir.backend.js.transformers.irToJs
 
-import org.jetbrains.kotlin.ir.backend.js.utils.getJsPolyfill
+import org.jetbrains.kotlin.ir.backend.js.utils.JsAnnotations
 import org.jetbrains.kotlin.ir.backend.js.utils.hasJsPolyfill
 import org.jetbrains.kotlin.ir.declarations.*
+import org.jetbrains.kotlin.ir.expressions.IrConst
+import org.jetbrains.kotlin.ir.util.getAnnotation
 import org.jetbrains.kotlin.js.backend.ast.JsStatement
 
 class JsPolyfills {
@@ -31,9 +33,11 @@ class JsPolyfills {
     private fun Iterable<IrDeclaration>.asImplementationList() =
         asSequence().asImplementationList()
 
-    private fun Sequence<IrDeclaration>.asImplementationList(): List<JsStatement> =
+    @Suppress("UNCHECKED_CAST")
-        map { it.getJsPolyfill()!! }
+    private fun Sequence<IrDeclaration>.asImplementationList(): List<JsStatement> {
-            .distinct()
+        return map { it to it.getAnnotation(JsAnnotations.JsPolyfillFqn)!!.getValueArgument(0)!! }
-            .flatMap { parseJsCode(it).orEmpty() } // FIXME!!!
+            .distinctBy { (it.second as IrConst<String>).value }
+            .flatMap { (container, polyfill) -> translateJsCodeIntoStatementList(polyfill, null, container).orEmpty() }
             .toList()
+    }
 }

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/transformers/irToJs/jsAstUtils.kt

@@ -7,10 +7,13 @@ package org.jetbrains.kotlin.ir.backend.js.transformers.irToJs
 
 import org.jetbrains.kotlin.backend.common.compilationException
 import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.IrFileEntry
 import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
+import org.jetbrains.kotlin.ir.backend.js.JsIrBackendContext
 import org.jetbrains.kotlin.ir.backend.js.JsStatementOrigins
 import org.jetbrains.kotlin.ir.backend.js.utils.*
 import org.jetbrains.kotlin.ir.declarations.IrClass
+import org.jetbrains.kotlin.ir.declarations.IrDeclaration
 import org.jetbrains.kotlin.ir.declarations.IrFunction
 import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
 import org.jetbrains.kotlin.ir.expressions.*
@@ -491,15 +494,22 @@ internal fun <T : JsNode> T.withSource(node: IrElement, context: JsGenerationCon
 private inline fun <T : JsNode> T.addSourceInfoIfNeed(node: IrElement, context: JsGenerationContext) {
     if (!context.staticContext.genSourcemaps) return
 
-    if (node.startOffset == UNDEFINED_OFFSET || node.endOffset == UNDEFINED_OFFSET) return
+    val sourceInfo = node.getSourceInfo(context.currentFile.fileEntry) ?: return
-
-    val fileEntry = context.currentFile.fileEntry
-
-    val path = fileEntry.name
-    val startLine = fileEntry.getLineNumber(node.startOffset)
-    val startColumn = fileEntry.getColumnNumber(node.startOffset)
 
     // TODO maybe it's better to fix in JsExpressionStatement
     val locationTarget = if (this is JsExpressionStatement) this.expression else this
-    locationTarget.source = JsLocation(path, startLine, startColumn)
+    locationTarget.source = sourceInfo
+}
+
+fun IrElement.getSourceInfo(container: IrDeclaration): JsLocation? {
+    val fileEntry = container.fileOrNull?.fileEntry ?: return null
+    return getSourceInfo(fileEntry)
+}
+
+fun IrElement.getSourceInfo(fileEntry: IrFileEntry): JsLocation? {
+    if (startOffset == UNDEFINED_OFFSET || endOffset == UNDEFINED_OFFSET) return null
+    val path = fileEntry.name
+    val startLine = fileEntry.getLineNumber(startOffset)
+    val startColumn = fileEntry.getColumnNumber(startOffset)
+    return JsLocation(path, startLine, startColumn)
 }

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/transformers/irToJs/jsCode.kt

@@ -8,27 +8,44 @@ package org.jetbrains.kotlin.ir.backend.js.transformers.irToJs
 import com.google.gwt.dev.js.ThrowExceptionOnErrorReporter
 import com.google.gwt.dev.js.rhino.CodePosition
 import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.IrFileEntry
 import org.jetbrains.kotlin.ir.backend.js.JsIrBackendContext
 import org.jetbrains.kotlin.ir.backend.js.lower.PropertyLazyInitLowering
+import org.jetbrains.kotlin.ir.declarations.IrDeclaration
 import org.jetbrains.kotlin.ir.declarations.IrVariable
 import org.jetbrains.kotlin.ir.expressions.*
+import org.jetbrains.kotlin.ir.util.fileOrNull
 import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
 import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
 import org.jetbrains.kotlin.ir.visitors.acceptVoid
-import org.jetbrains.kotlin.js.backend.ast.JsFunctionScope
+import org.jetbrains.kotlin.js.backend.ast.*
-import org.jetbrains.kotlin.js.backend.ast.JsProgram
-import org.jetbrains.kotlin.js.backend.ast.JsRootScope
-import org.jetbrains.kotlin.js.backend.ast.JsStatement
 import org.jetbrains.kotlin.js.parser.parseExpressionOrStatement
 
-// Returns null if constant expression could not be parsed
+/**
-fun translateJsCodeIntoStatementList(code: IrExpression, context: JsIrBackendContext): List<JsStatement>? {
+ * Returns null if constant expression could not be parsed.
-    // TODO: support proper symbol linkage and label clash resolution
+ */
-    // FIXME: Provide debug info
+fun translateJsCodeIntoStatementList(code: IrExpression, context: JsIrBackendContext?, container: IrDeclaration) =
-    return parseJsCode(foldString(code, context) ?: return null)
+    translateJsCodeIntoStatementList(
-}
+        code,
+        context,
+        code.getSourceInfo(container) ?: container.fileOrNull?.fileEntry?.let { JsLocation(it.name, 0, 0) }
+    )
+
+/**
+ * Returns null if constant expression could not be parsed.
+ */
+fun translateJsCodeIntoStatementList(code: IrExpression, context: JsIrBackendContext?, fileEntry: IrFileEntry) =
+    translateJsCodeIntoStatementList(code, context, code.getSourceInfo(fileEntry) ?: JsLocation(fileEntry.name, 0, 0))
+
+private fun translateJsCodeIntoStatementList(
+    code: IrExpression,
+    context: JsIrBackendContext?,
+    sourceInfo: JsLocation?
+): List<JsStatement>? {
+    // TODO: support proper symbol linkage and label clash resolution
+    val (fileName, startLine, offset) = sourceInfo ?: JsLocation("<js-code>", 0, 0)
+    val jsCode = foldString(code, context) ?: return null
 
-fun parseJsCode(jsCode: String): List<JsStatement>? {
     // Parser can change local or global scope.
     // In case of js we want to keep new local names,
     // but no new global ones.
@@ -36,12 +53,17 @@ fun parseJsCode(jsCode: String): List<JsStatement>? {
     val temporaryRootScope = JsRootScope(JsProgram())
     val currentScope = JsFunctionScope(temporaryRootScope, "js")
 
-    // TODO: write debug info, see how it's done in CallExpressionTranslator.parseJsCode
+    // NOTE: emitting the correct debug info for JS injections is a non-trivial task, because the injection may consist of
-
+    // constant-evaluated strings, whose origin is hard to track in the JS parser. Also, the presence of explicit \n characters in
-    return parseExpressionOrStatement(jsCode, ThrowExceptionOnErrorReporter, currentScope, CodePosition(0, 0), "<js-code>")
+    // the JS string literal breaks the debug info, because at this level we are unable to distinguish multiline string literals and
+    // single-line string literals with explicit \n in it.
+    //
+    // So we try to generate the debug info on the best-effort basis. It should work correctly with plain string literals without
+    // concatenations, interpolations or backslash replacements like \n.
+    return parseExpressionOrStatement(jsCode, ThrowExceptionOnErrorReporter, currentScope, CodePosition(startLine, offset), fileName)
 }
 
-fun foldString(expression: IrExpression, context: JsIrBackendContext): String? {
+fun foldString(expression: IrExpression, context: JsIrBackendContext?): String? {
     val builder = StringBuilder()
     var foldingFailed = false
     expression.acceptVoid(object : IrElementVisitorVoid {
@@ -65,7 +87,7 @@ fun foldString(expression: IrExpression, context: JsIrBackendContext): String? {
         override fun visitGetField(expression: IrGetField) {
             val owner = expression.symbol.owner
             owner.initializer?.expression?.acceptVoid(this)
-                ?: context.fieldToInitializer[owner]?.acceptVoid(this)
+                ?: context?.fieldToInitializer?.get(owner)?.acceptVoid(this)
         }
 
         override fun visitCall(expression: IrCall) {
@@ -101,12 +123,12 @@ fun foldString(expression: IrExpression, context: JsIrBackendContext): String? {
     return builder.toString()
 }
 
-private class InitFunVisitor(private val context: JsIrBackendContext) : IrElementVisitorVoid {
+private class InitFunVisitor(private val context: JsIrBackendContext?) : IrElementVisitorVoid {
     override fun visitElement(element: IrElement) {
         element.acceptChildrenVoid(this)
     }
 
     override fun visitSetField(expression: IrSetField) {
-        context.fieldToInitializer[expression.symbol.owner] = expression.value
+        context?.fieldToInitializer?.set(expression.symbol.owner, expression.value)
     }
 }

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/utils/AnnotationUtils.kt

@@ -51,9 +51,6 @@ fun IrAnnotationContainer.getJsQualifier(): String? =
 fun IrAnnotationContainer.getJsName(): String? =
     getAnnotation(JsAnnotations.jsNameFqn)?.getSingleConstStringArgument()
 
-fun IrAnnotationContainer.getJsPolyfill(): String? =
-    getAnnotation(JsAnnotations.JsPolyfillFqn)?.getSingleConstStringArgument()
-
 fun IrAnnotationContainer.hasJsPolyfill(): Boolean =
     hasAnnotation(JsAnnotations.JsPolyfillFqn)
 

compiler/testData/debug/stepping/jsCode.kt

@@ -0,0 +1,80 @@
+// TARGET_BACKEND: JS_IR
+
+// FILE: a.kt
+@JsExport
+fun exclamate(a: String) =
+    "$a!"
+
+@Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE")
+@JsFun("""
+    function (a, b) {
+        return _.exclamate(a) +
+            _.exclamate(b);
+    }
+""")
+external fun exclamateAndConcat(a: String, b: String): String
+
+@Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE")
+@JsPolyfill("""
+if (typeof String.prototype.myAwesomePolyfill === "undefined") {
+    Object.defineProperty(String.prototype, "myAwesomePolyfill", {
+        value: function () {
+            return this + "!";
+        }
+    });
+}
+""")
+internal inline fun String.myAwesomePolyfill(): Boolean =
+    asDynamic()
+        .myAwesomePolyfill()
+
+val walter1VarDecl = "var walter1 = _.exclamate('Walter');"
+val jesse1VarDecl = "var jesse1 = _.exclamate(jesse);"
+
+// FILE: test.kt
+
+fun noop() {}
+
+fun box() {
+    noop()
+    exclamateAndConcat(
+        "hello",
+        "world")
+    val jesse = "Jesse"
+    js(
+        "_.exclamate(jesse);") // Local variable is captured
+    js(
+        "_.exclamate('Walter');") // No local variables captured
+    js(
+        walter1VarDecl +
+        "_.exclamate(walter1);")
+    js(
+        jesse1VarDecl +
+                "_.exclamate(jesse1);")
+    
+    "foo".myAwesomePolyfill()
+}
+
+// EXPECTATIONS
+// test.kt:39 box
+// test.kt:41 box
+// test.kt:42 box
+// a.kt:11 box
+// a.kt:6 exclamate
+// a.kt:12 box
+// a.kt:6 exclamate
+// test.kt:43 box
+// test.kt:45 box
+// a.kt:6 exclamate
+// test.kt:47 box
+// a.kt:6 exclamate
+// test.kt:49 box
+// a.kt:6 exclamate
+// test.kt:49 box
+// a.kt:6 exclamate
+// test.kt:52 box
+// a.kt:6 exclamate
+// test.kt:52 box
+// a.kt:6 exclamate
+// a.kt:29 box
+// a.kt:22 value

js/js.parser/src/com/google/gwt/dev/js/rhino/LineBuffer.java

@@ -67,6 +67,7 @@ final class LineBuffer {
     LineBuffer(Reader in, CodePosition position) {
         this.in = in;
         this.lineno = position.getLine();
+        this.columnno = position.getOffset();
         this.lineStart = -position.getOffset();
     }
 
@@ -77,14 +78,16 @@ final class LineBuffer {
 
             int c = buffer[offset];
             ++offset;
+            prevColumnno = columnno++;
 
             if ((c & EOL_HINT_MASK) == 0) {
                 switch (c) {
                     case '\r':
                         // if the next character is a newline, skip past it.
                         if (offset != end) {
-                            if (buffer[offset] == '\n')
+                            if (buffer[offset] == '\n') {
                                 ++offset;
+                            }
                         } else {
                             // set a flag for fill(), in case the first char
                             // of the next fill is a newline.
@@ -95,6 +98,8 @@ final class LineBuffer {
                         prevStart = lineStart;
                         lineStart = offset;
                         lineno++;
+                        prevColumnno = columnno;
+                        columnno = 0;
                         return '\n';
                 }
             }
@@ -120,10 +125,12 @@ final class LineBuffer {
         if (offset == 0) // Same as if (hitEOF)
             return;
         offset--;
+        columnno--;
         int c = buffer[offset];
         if ((c & EOL_HINT_MASK) == 0 && isEndOfLine(c)) {
             lineStart = prevStart;
             lineno--;
+            columnno = prevColumnno;
         }
     }
 
@@ -147,6 +154,7 @@ final class LineBuffer {
         }
 
         ++offset;
+        prevColumnno = columnno++;
     }
 
     int peek() throws IOException {
@@ -182,6 +190,7 @@ final class LineBuffer {
             int c = buffer[offset];
             if (test == c) {
                 ++offset;
+                prevColumnno = columnno++;
                 return true;
             }
             if (c < 128 || !formatChar(c)) {
@@ -292,7 +301,8 @@ final class LineBuffer {
         // likely get to EOF for real) by doing yet another fill().
         if (lastWasCR) {
             if (buffer[0] == '\n') {
-              offset++;
+              ++offset;
+              prevColumnno = columnno++;
               if (end == 1)
                   return fill();
             }
@@ -303,6 +313,9 @@ final class LineBuffer {
     }
 
     int getLineno() { return lineno; }
+
+    int getColumnno() { return columnno; };
+
     boolean eof() { return hitEOF; }
 
     private static boolean formatChar(int c) {
@@ -322,6 +335,8 @@ final class LineBuffer {
     private int end = 0;
     private int otherEnd;
     private int lineno;
+    private int columnno;
+    private int prevColumnno = 0;
 
     private int lineStart = 0;
     private int otherStart = 0;

js/js.parser/src/com/google/gwt/dev/js/rhino/TokenStream.java

@@ -503,6 +503,7 @@ public class TokenStream {
         flags = 0;
         secondToLastPosition = position;
         lastPosition = position;
+        tokenPosition = position;
         lastTokenPosition = position;
     }
 
@@ -624,7 +625,7 @@ public class TokenStream {
             }
         } while (isJSSpace(c) || c == '\n');
 
-        tokenPosition = new CodePosition(in.getLineno(), Math.max(in.getOffset() - 1, 0));
+        tokenPosition = new CodePosition(in.getLineno(), in.getColumnno() - 1);
         if (c == EOF_CHAR)
             return EOF;
         if (c != '-' && c != '\n')

js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/ir/IrJsSteppingTestGenerated.java

@@ -247,6 +247,12 @@ public class IrJsSteppingTestGenerated extends AbstractIrJsSteppingTest {
         runTest("compiler/testData/debug/stepping/inlineSimpleCall.kt");
     }
 
+    @Test
+    @TestMetadata("jsCode.kt")
+    public void testJsCode() throws Exception {
+        runTest("compiler/testData/debug/stepping/jsCode.kt");
+    }
+
     @Test
     @TestMetadata("kt15259.kt")
     public void testKt15259() throws Exception {

js/js.translator/testData/lineNumbers/jsCode.kt

@@ -5,7 +5,5 @@ fun foo() {
     println("after: $x")
 }
 
-// LINES(JS):    1 6 2 2 3 3 4 4     5 5
+// LINES(JS):    1 6 2 2 3 3 4 4 5 5
-// LINES(JS_IR):     2 2 3 3     2 2 5 5
+// LINES(JS_IR):     2 2 3 3 4 4 5 5
-// FIXME:                    ^^^^^^^
-//                           js function call debug info is incorrect