azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor generator of JS source map

Browse Source 
- refactor pipeline for generation of source map
- generate "empty" mappings for nodes that impossible
  to map to something reasonable
- generate more accurate locations in source maps for specific
  JS AST nodes
- for binary operation nodes parser now returns location
  of binary operator tokens instead of location of first operand
- change completely how source map remapper works
This commit is contained in:
Alexey Andreev
2017-06-20 15:34:36 +03:00
parent 6a2f3d8065
commit dea73ebecf
30 changed files with 622 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/js.parser/src/com/google/gwt/dev/js/JsAstMapper.java
+4 -1
View File
@@ -1115,7 +1115,10 @@ public class JsAstMapper {
astNode.setSource(jsLocation);
}
else if (astNode instanceof JsExpressionStatement) {
((JsExpressionStatement) astNode).getExpression().setSource(jsLocation);
JsExpression expression = ((JsExpressionStatement) astNode).getExpression();
if (expression.getSource() == null) {
expression.setSource(jsLocation);
}
}
}
return astNode;
js/js.parser/src/com/google/gwt/dev/js/rhino/Parser.java
+21 -24
View File
@@ -180,7 +180,7 @@ public class Parser extends Observable {
// by '(', assume <name> starts memberExpr
Node memberExprHead = nameNode;
nameNode = null;
memberExprNode = memberExprTail(ts, false, memberExprHead, basePosition);
memberExprNode = memberExprTail(ts, false, memberExprHead);
}
mustMatchToken(ts, TokenStream.LP, "msg.no.paren.parms");
}
@@ -708,20 +708,20 @@ public class Parser extends Observable {
}
private Node expr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = assignExpr(ts, inForInit);
while (ts.matchToken(TokenStream.COMMA)) {
CodePosition position = ts.tokenPosition;
pn = nf.createBinary(TokenStream.COMMA, pn, assignExpr(ts, inForInit), position);
}
return pn;
}
private Node assignExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = condExpr(ts, inForInit);
if (ts.matchToken(TokenStream.ASSIGN)) {
// omitted: "invalid assignment left-hand side" check.
CodePosition position = ts.tokenPosition;
pn = nf.createBinary(TokenStream.ASSIGN, ts.getOp(), pn, assignExpr(ts, inForInit), position);
}
@@ -729,10 +729,10 @@ public class Parser extends Observable {
}
private Node condExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = orExpr(ts, inForInit);
if (ts.matchToken(TokenStream.HOOK)) {
CodePosition position = ts.tokenPosition;
Node ifTrue = assignExpr(ts, false);
mustMatchToken(ts, TokenStream.COLON, "msg.no.colon.cond");
Node ifFalse = assignExpr(ts, inForInit);
@@ -743,9 +743,9 @@ public class Parser extends Observable {
}
private Node orExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = andExpr(ts, inForInit);
while (ts.matchToken(TokenStream.OR)) {
CodePosition position = ts.tokenPosition;
pn = nf.createBinary(TokenStream.OR, pn, andExpr(ts, inForInit), position);
}
@@ -753,9 +753,9 @@ public class Parser extends Observable {
}
private Node andExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = bitOrExpr(ts, inForInit);
while (ts.matchToken(TokenStream.AND)) {
CodePosition position = ts.tokenPosition;
pn = nf.createBinary(TokenStream.AND, pn, bitOrExpr(ts, inForInit), position);
}
@@ -763,44 +763,46 @@ public class Parser extends Observable {
}
private Node bitOrExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = bitXorExpr(ts, inForInit);
while (ts.matchToken(TokenStream.BITOR)) {
CodePosition position = ts.tokenPosition;
pn = nf.createBinary(TokenStream.BITOR, pn, bitXorExpr(ts, inForInit), position);
}
return pn;
}
private Node bitXorExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = bitAndExpr(ts, inForInit);
while (ts.matchToken(TokenStream.BITXOR)) {
CodePosition position = ts.tokenPosition;
pn = nf.createBinary(TokenStream.BITXOR, pn, bitAndExpr(ts, inForInit), position);
}
return pn;
}
private Node bitAndExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = eqExpr(ts, inForInit);
while (ts.matchToken(TokenStream.BITAND)) {
CodePosition position = ts.tokenPosition;
pn = nf.createBinary(TokenStream.BITAND, pn, eqExpr(ts, inForInit), position);
}
return pn;
}
private Node eqExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = relExpr(ts, inForInit);
while (ts.matchToken(TokenStream.EQOP)) {
CodePosition position = ts.tokenPosition;
pn = nf.createBinary(TokenStream.EQOP, ts.getOp(), pn, relExpr(ts, inForInit), position);
}
return pn;
}
private Node relExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = shiftExpr(ts);
CodePosition position = ts.tokenPosition;
while (ts.matchToken(TokenStream.RELOP)) {
int op = ts.getOp();
if (inForInit && op == TokenStream.IN) {
@@ -809,26 +811,26 @@ public class Parser extends Observable {
}
pn = nf.createBinary(TokenStream.RELOP, op, pn, shiftExpr(ts), position);
position = ts.tokenPosition;
}
return pn;
}
private Node shiftExpr(TokenStream ts) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = addExpr(ts);
while (ts.matchToken(TokenStream.SHOP)) {
CodePosition position = ts.tokenPosition;
pn = nf.createBinary(TokenStream.SHOP, ts.getOp(), pn, addExpr(ts), position);
}
return pn;
}
private Node addExpr(TokenStream ts) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
int tt;
Node pn = mulExpr(ts);
while ((tt = ts.getToken()) == TokenStream.ADD || tt == TokenStream.SUB) {
// flushNewLines
CodePosition position = ts.tokenPosition;
pn = nf.createBinary(tt, pn, mulExpr(ts), position);
}
ts.ungetToken(tt);
@@ -837,13 +839,13 @@ public class Parser extends Observable {
}
private Node mulExpr(TokenStream ts) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
int tt;
Node pn = unaryExpr(ts);
while ((tt = ts.peekToken()) == TokenStream.MUL || tt == TokenStream.DIV || tt == TokenStream.MOD) {
tt = ts.getToken();
CodePosition position = ts.tokenPosition;
pn = nf.createBinary(tt, pn, unaryExpr(ts), position);
}
@@ -894,6 +896,7 @@ public class Parser extends Observable {
if (((peeked = ts.peekToken()) == TokenStream.INC || peeked == TokenStream.DEC)
&& ts.getLineno() == lineno) {
int pf = ts.getToken();
position = ts.tokenPosition;
return nf.createUnary(pf, TokenStream.POST, pn, position);
}
return pn;
@@ -960,16 +963,17 @@ public class Parser extends Observable {
pn = primaryExpr(ts);
}
return memberExprTail(ts, allowCallSyntax, pn, position);
return memberExprTail(ts, allowCallSyntax, pn);
}
private Node memberExprTail(
TokenStream ts, boolean allowCallSyntax,
Node pn, CodePosition position
Node pn
) throws IOException, JavaScriptException {
lastExprEndLine = ts.getLineno();
int tt;
while ((tt = ts.getToken()) > TokenStream.EOF) {
CodePosition position = ts.tokenPosition;
if (tt == TokenStream.DOT) {
ts.treatKeywordAsIdentifier = true;
mustMatchToken(ts, TokenStream.NAME, "msg.no.name.after.dot");
@@ -1144,13 +1148,6 @@ public class Parser extends Observable {
return null; // should never reach here
}
private static CodePosition getNextTokenPosition(TokenStream ts) throws IOException, JavaScriptException {
int tt = ts.getToken();
CodePosition result = ts.tokenPosition;
ts.ungetToken(tt);
return result;
}
private int lastExprEndLine; // Hack to handle function expr termination.
private final IRFactory nf;
private boolean ok; // Did the parse encounter an error?
js/js.parser/src/org/jetbrains/kotlin/js/parser/sourcemaps/SourceMap.kt
+12 -1
View File
@@ -16,15 +16,26 @@
package org.jetbrains.kotlin.js.parser.sourcemaps
import java.io.PrintStream
import java.io.Reader
class SourceMap(val sourceContentResolver: (String) -> Reader?) {
val groups = mutableListOf<SourceMapGroup>()
fun debug(writer: PrintStream = System.out) {
for ((index, group) in groups.withIndex()) {
writer.print("${index + 1}:")
for (segment in group.segments) {
writer.print(" ${segment.generatedColumnNumber + 1}:${segment.sourceLineNumber + 1},${segment.sourceColumnNumber + 1}")
}
writer.println()
}
}
}
class SourceMapSegment(
val generatedColumnNumber: Int,
val sourceFileName: String,
val sourceFileName: String?,
val sourceLineNumber: Int,
val sourceColumnNumber: Int
)
js/js.parser/src/org/jetbrains/kotlin/js/parser/sourcemaps/SourceMapLocationRemapper.kt
+98 -31
View File
@@ -18,51 +18,118 @@ package org.jetbrains.kotlin.js.parser.sourcemaps
import org.jetbrains.kotlin.js.backend.ast.*
class SourceMapLocationRemapper(val sourceMaps: Map<String, SourceMap>) {
class SourceMapLocationRemapper(private val sourceMap: SourceMap) {
fun remap(node: JsNode) {
node.accept(visitor)
val listCollector = JsNodeFlatListCollector()
node.accept(listCollector)
applySourceMap(listCollector.nodeList)
}
private val visitor = object : RecursiveJsVisitor() {
private var lastSourceMap: SourceMap? = null
private var lastGroup: SourceMapGroup? = null
private var lastSegmentIndex = 0
private fun applySourceMap(nodes: List<JsNode>) {
var lastGroup: SourceMapGroup? = null
var lastGroupIndex = 0
var lastSegment: SourceMapSegment? = null
var lastSegmentIndex = 0
override fun visitElement(node: JsNode) {
if (node is SourceInfoAwareJsNode) {
if (!remapNode(node)) {
node.source = null
fun findCorrespondingSegment(node: SourceInfoAwareJsNode): SourceMapSegment? {
val source = node.source as? JsLocation ?: return null
val group = sourceMap.groups.getOrElse(source.startLine) { return null }
if (lastGroup != group) {
if (lastGroup != null) {
val segmentsToSkip = lastGroup!!.segments.drop(lastSegmentIndex).toMutableList()
if (lastGroupIndex + 1 < source.startLine) {
segmentsToSkip += sourceMap.groups.subList((lastGroupIndex + 1), source.startLine).flatMap { it.segments }
}
segmentsToSkip.lastOrNull()?.let { lastSegment = it }
}
lastGroup = group
lastGroupIndex = source.startLine
lastSegmentIndex = 0
}
super.visitElement(node)
while (lastSegmentIndex < group.segments.size) {
val segment = group.segments[lastSegmentIndex]
if (segment.generatedColumnNumber > source.startChar) break
lastSegment = segment
lastSegmentIndex++
}
return lastSegment
}
private fun remapNode(node: SourceInfoAwareJsNode): Boolean {
val source = node.source as? JsLocation ?: return false
val sourceMap = sourceMaps[source.file] ?: return false
val group = sourceMap.groups.getOrElse(source.startLine) { return false }
if (group.segments.isEmpty()) return false
if (lastSourceMap != sourceMap || lastGroup != group) {
lastSegmentIndex = 0
for (node in nodes.asSequence().filterIsInstance<SourceInfoAwareJsNode>()) {
val segment = findCorrespondingSegment(node)
val sourceFileName = segment?.sourceFileName
node.source = if (sourceFileName != null) {
val location = JsLocation(segment.sourceFileName, segment.sourceLineNumber, segment.sourceColumnNumber)
JsLocationWithEmbeddedSource(location, sourceMap) { sourceMap.sourceContentResolver(segment.sourceFileName) }
}
if (group.segments[lastSegmentIndex].generatedColumnNumber > source.startChar) {
if (lastSegmentIndex == 0) return false
lastSegmentIndex = 0
else {
null
}
}
}
while (lastSegmentIndex + 1 < group.segments.size) {
val nextIndex = lastSegmentIndex + 1
if (group.segments[nextIndex].generatedColumnNumber > source.startChar) break
lastSegmentIndex = nextIndex
internal class JsNodeFlatListCollector : RecursiveJsVisitor() {
val nodeList = mutableListOf<JsNode>()
override fun visitDoWhile(x: JsDoWhile) {
nodeList += x
accept(x.body)
accept(x.condition)
}
override fun visitBinaryExpression(x: JsBinaryOperation) = handleNode(x, x.arg1, x.arg2)
override fun visitConditional(x: JsConditional) = handleNode(x, x.testExpression, x.thenExpression, x.elseExpression)
override fun visitArrayAccess(x: JsArrayAccess) = handleNode(x, x.arrayExpression, x.indexExpression)
override fun visitArray(x: JsArrayLiteral) = handleNode(x, *x.expressions.toTypedArray())
override fun visitPrefixOperation(x: JsPrefixOperation) = handleNode(x, x.arg)
override fun visitPostfixOperation(x: JsPostfixOperation) = handleNode(x, x.arg)
override fun visitNameRef(nameRef: JsNameRef) = handleNode(nameRef, nameRef.qualifier)
override fun visitInvocation(invocation: JsInvocation) =
handleNode(invocation, invocation.qualifier, *invocation.arguments.toTypedArray())
override fun visitElement(node: JsNode) {
nodeList += node
node.acceptChildren(this)
}
private fun handleNode(node: JsNode, vararg children: JsNode?) {
val nonNullChildren = children.mapNotNull { it }
if (nonNullChildren.isEmpty()) {
nodeList += node
}
else {
val firstChild = nonNullChildren.first()
if (node.isNotBefore(firstChild)) {
accept(firstChild)
nodeList += node
nonNullChildren.drop(1).forEach { accept(it) }
}
else {
nodeList += node
nonNullChildren.forEach { accept(it) }
}
}
}
val segment = group.segments[lastSegmentIndex]
val location = JsLocation(segment.sourceFileName, segment.sourceLineNumber, segment.sourceColumnNumber)
node.source = JsLocationWithEmbeddedSource(location, sourceMap) { sourceMap.sourceContentResolver(segment.sourceFileName) }
return true
private fun JsNode.isNotBefore(other: JsNode): Boolean {
val first = (source as? JsLocation ?: return false)
val second = (other.source as? JsLocation ?: return false)
if (first.file != second.file) return false
return first.startLine > second.startLine || (first.startLine == second.startLine && first.startChar >= second.startChar)
}
}
}
js/js.parser/src/org/jetbrains/kotlin/js/parser/sourcemaps/SourceMapParser.kt
+7 -21
View File
@@ -14,22 +14,6 @@
* limitations under the License.
*/
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.js.parser.sourcemaps
import org.json.JSONArray
@@ -115,13 +99,15 @@ object SourceMapParser {
if (stream.isEncodedInt) {
stream.readInt() ?: return stream.createError("VLQ-encoded name index expected")
}
}
if (sourceIndex !in sources.indices) {
return stream.createError("Source index $sourceIndex is out of bounds ${sources.indices}")
if (sourceIndex !in sources.indices) {
return stream.createError("Source index $sourceIndex is out of bounds ${sources.indices}")
}
currentGroup.segments += SourceMapSegment(jsColumn, sourceRoot + sources[sourceIndex], sourceLine, sourceColumn)
}
else {
currentGroup.segments += SourceMapSegment(jsColumn, null, -1, -1)
}
currentGroup.segments += SourceMapSegment(jsColumn, sourceRoot + sources[sourceIndex], sourceLine, sourceColumn)
when {
stream.isEof -> return stream.createError("Unexpected EOF, ',' or ';' expected")