azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add test infrastructure for JS source map parser and generator

Browse Source
This commit is contained in:
Alexey Andreev
2017-05-23 12:24:16 +03:00
parent ca738a9536
commit c5bf76578a
9 changed files with 156 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/js.ast/src/org/jetbrains/kotlin/js/backend/JsToStringGenerationVisitor.java
-4
View File
@@ -608,13 +608,9 @@ public class JsToStringGenerationVisitor extends JsVisitor {
}
p.maybeIndent();
beforeNodePrinted(nameRef);
p.print(nameRef.getIdent());
}
protected void beforeNodePrinted(JsNode node) {
}
@Override
public void visitNew(@NotNull JsNew x) {
p.print(CHARS_NEW);
js/js.ast/src/org/jetbrains/kotlin/js/backend/ast/JsObjectLiteral.java
+7 -3
View File
@@ -13,20 +13,24 @@ import java.util.List;
public final class JsObjectLiteral extends JsLiteral {
private final List<JsPropertyInitializer> properties;
private final boolean multiline;
private boolean multiline;
public JsObjectLiteral() {
this(new SmartList<JsPropertyInitializer>());
this(new SmartList<>());
}
public JsObjectLiteral(boolean multiline) {
this(new SmartList<JsPropertyInitializer>(), multiline);
this(new SmartList<>(), multiline);
}
public boolean isMultiline() {
return multiline;
}
public void setMultiline(boolean multiline) {
this.multiline = multiline;
}
public JsObjectLiteral(List<JsPropertyInitializer> properties) {
this(properties, false);
}
js/js.ast/src/org/jetbrains/kotlin/js/backend/ast/JsVars.java
+10 -5
View File
@@ -18,14 +18,14 @@ import java.util.*;
public class JsVars extends SourceInfoAwareJsNode implements JsStatement, Iterable<JsVars.JsVar> {
private final List<JsVar> vars;
private final boolean multiline;
private boolean multiline;
public JsVars() {
this(new SmartList<JsVar>(), false);
this(new SmartList<>(), false);
}
public JsVars(boolean multiline) {
this(new SmartList<JsVar>(), multiline);
this(new SmartList<>(), multiline);
}
public JsVars(List<JsVar> vars, boolean multiline) {
@@ -34,17 +34,21 @@ public class JsVars extends SourceInfoAwareJsNode implements JsStatement, Iterab
}
public JsVars(JsVar var) {
this(new SmartList<JsVar>(var), false);
this(new SmartList<>(var), false);
}
public JsVars(JsVar... vars) {
this(new SmartList<JsVar>(vars), false);
this(new SmartList<>(vars), false);
}
public boolean isMultiline() {
return multiline;
}
public void setMultiline(boolean multiline) {
this.multiline = multiline;
}
/**
* A var declared using the JavaScript <code>var</code> statement.
*/
@@ -138,6 +142,7 @@ public class JsVars extends SourceInfoAwareJsNode implements JsStatement, Iterab
}
@Override
@NotNull
public Iterator<JsVar> iterator() {
return vars.iterator();
}
js/js.parser/src/com/google/gwt/dev/js/rhino/Parser.java
+33 -13
View File
@@ -708,88 +708,98 @@ public class Parser extends Observable {
}
private Node expr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = assignExpr(ts, inForInit);
while (ts.matchToken(TokenStream.COMMA)) {
pn = nf.createBinary(TokenStream.COMMA, pn, assignExpr(ts, inForInit), pn.getPosition());
pn = nf.createBinary(TokenStream.COMMA, pn, assignExpr(ts, inForInit), position);
}
return pn;
}
private Node assignExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = condExpr(ts, inForInit);
if (ts.matchToken(TokenStream.ASSIGN)) {
// omitted: "invalid assignment left-hand side" check.
pn = nf.createBinary(TokenStream.ASSIGN, ts.getOp(), pn, assignExpr(ts, inForInit), pn.getPosition());
pn = nf.createBinary(TokenStream.ASSIGN, ts.getOp(), pn, assignExpr(ts, inForInit), position);
}
return pn;
}
private Node condExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = orExpr(ts, inForInit);
if (ts.matchToken(TokenStream.HOOK)) {
Node ifTrue = assignExpr(ts, false);
mustMatchToken(ts, TokenStream.COLON, "msg.no.colon.cond");
Node ifFalse = assignExpr(ts, inForInit);
return nf.createTernary(pn, ifTrue, ifFalse, pn.getPosition());
return nf.createTernary(pn, ifTrue, ifFalse, position);
}
return pn;
}
private Node orExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = andExpr(ts, inForInit);
while (ts.matchToken(TokenStream.OR)) {
pn = nf.createBinary(TokenStream.OR, pn, andExpr(ts, inForInit), pn.getPosition());
pn = nf.createBinary(TokenStream.OR, pn, andExpr(ts, inForInit), position);
}
return pn;
}
private Node andExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = bitOrExpr(ts, inForInit);
while (ts.matchToken(TokenStream.AND)) {
pn = nf.createBinary(TokenStream.AND, pn, bitOrExpr(ts, inForInit), pn.getPosition());
pn = nf.createBinary(TokenStream.AND, pn, bitOrExpr(ts, inForInit), position);
}
return pn;
}
private Node bitOrExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = bitXorExpr(ts, inForInit);
while (ts.matchToken(TokenStream.BITOR)) {
pn = nf.createBinary(TokenStream.BITOR, pn, bitXorExpr(ts, inForInit), pn.getPosition());
pn = nf.createBinary(TokenStream.BITOR, pn, bitXorExpr(ts, inForInit), position);
}
return pn;
}
private Node bitXorExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = bitAndExpr(ts, inForInit);
while (ts.matchToken(TokenStream.BITXOR)) {
pn = nf.createBinary(TokenStream.BITXOR, pn, bitAndExpr(ts, inForInit), pn.getPosition());
pn = nf.createBinary(TokenStream.BITXOR, pn, bitAndExpr(ts, inForInit), position);
}
return pn;
}
private Node bitAndExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = eqExpr(ts, inForInit);
while (ts.matchToken(TokenStream.BITAND)) {
pn = nf.createBinary(TokenStream.BITAND, pn, eqExpr(ts, inForInit), pn.getPosition());
pn = nf.createBinary(TokenStream.BITAND, pn, eqExpr(ts, inForInit), position);
}
return pn;
}
private Node eqExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = relExpr(ts, inForInit);
while (ts.matchToken(TokenStream.EQOP)) {
pn = nf.createBinary(TokenStream.EQOP, ts.getOp(), pn, relExpr(ts, inForInit), pn.getPosition());
pn = nf.createBinary(TokenStream.EQOP, ts.getOp(), pn, relExpr(ts, inForInit), position);
}
return pn;
}
private Node relExpr(TokenStream ts, boolean inForInit) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = shiftExpr(ts);
while (ts.matchToken(TokenStream.RELOP)) {
int op = ts.getOp();
@@ -798,26 +808,28 @@ public class Parser extends Observable {
break;
}
pn = nf.createBinary(TokenStream.RELOP, op, pn, shiftExpr(ts), pn.getPosition());
pn = nf.createBinary(TokenStream.RELOP, op, pn, shiftExpr(ts), position);
}
return pn;
}
private Node shiftExpr(TokenStream ts) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
Node pn = addExpr(ts);
while (ts.matchToken(TokenStream.SHOP)) {
pn = nf.createBinary(TokenStream.SHOP, ts.getOp(), pn, addExpr(ts), pn.getPosition());
pn = nf.createBinary(TokenStream.SHOP, ts.getOp(), pn, addExpr(ts), position);
}
return pn;
}
private Node addExpr(TokenStream ts) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
int tt;
Node pn = mulExpr(ts);
while ((tt = ts.getToken()) == TokenStream.ADD || tt == TokenStream.SUB) {
// flushNewLines
pn = nf.createBinary(tt, pn, mulExpr(ts), pn.getPosition());
pn = nf.createBinary(tt, pn, mulExpr(ts), position);
}
ts.ungetToken(tt);
@@ -825,13 +837,14 @@ public class Parser extends Observable {
}
private Node mulExpr(TokenStream ts) throws IOException, JavaScriptException {
CodePosition position = getNextTokenPosition(ts);
int tt;
Node pn = unaryExpr(ts);
while ((tt = ts.peekToken()) == TokenStream.MUL || tt == TokenStream.DIV || tt == TokenStream.MOD) {
tt = ts.getToken();
pn = nf.createBinary(tt, pn, unaryExpr(ts), pn.getPosition());
pn = nf.createBinary(tt, pn, unaryExpr(ts), position);
}
return pn;
@@ -1133,6 +1146,13 @@ public class Parser extends Observable {
return null; // should never reach here
}
private static CodePosition getNextTokenPosition(TokenStream ts) throws IOException, JavaScriptException {
int tt = ts.getToken();
CodePosition result = ts.tokenPosition;
ts.ungetToken(tt);
return result;
}
private int lastExprEndLine; // Hack to handle function expr termination.
private final IRFactory nf;
private boolean ok; // Did the parse encounter an error?
js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt
+62 -15
View File
@@ -16,6 +16,7 @@
package org.jetbrains.kotlin.js.test
import com.google.gwt.dev.js.ThrowExceptionOnErrorReporter
import com.intellij.openapi.util.io.FileUtil
import com.intellij.openapi.util.text.StringUtil
import com.intellij.openapi.vfs.StandardFileSystems
@@ -31,19 +32,19 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
import org.jetbrains.kotlin.config.*
import org.jetbrains.kotlin.idea.KotlinFileType
import org.jetbrains.kotlin.js.JavaScript
import org.jetbrains.kotlin.js.backend.ast.JsProgram
import org.jetbrains.kotlin.js.backend.ast.*
import org.jetbrains.kotlin.js.config.EcmaVersion
import org.jetbrains.kotlin.js.config.JSConfigurationKeys
import org.jetbrains.kotlin.js.config.JsConfig
import org.jetbrains.kotlin.js.dce.DeadCodeElimination
import org.jetbrains.kotlin.js.dce.InputFile
import org.jetbrains.kotlin.js.facade.K2JSTranslator
import org.jetbrains.kotlin.js.facade.MainCallParameters
import org.jetbrains.kotlin.js.facade.TranslationResult
import org.jetbrains.kotlin.js.facade.TranslationUnit
import org.jetbrains.kotlin.js.test.utils.DirectiveTestUtils
import org.jetbrains.kotlin.js.test.utils.JsTestUtils
import org.jetbrains.kotlin.js.test.utils.verifyAst
import org.jetbrains.kotlin.js.facade.*
import org.jetbrains.kotlin.js.parser.parse
import org.jetbrains.kotlin.js.parser.sourcemaps.*
import org.jetbrains.kotlin.js.sourceMap.JsSourceGenerationVisitor
import org.jetbrains.kotlin.js.sourceMap.SourceMap3Builder
import org.jetbrains.kotlin.js.test.utils.*
import org.jetbrains.kotlin.js.util.TextOutputImpl
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.psi.KtNamedFunction
import org.jetbrains.kotlin.psi.KtPsiFactory
@@ -55,10 +56,7 @@ import org.jetbrains.kotlin.test.KotlinTestUtils.TestFileFactory
import org.jetbrains.kotlin.test.KotlinTestWithEnvironment
import org.jetbrains.kotlin.test.TargetBackend
import org.jetbrains.kotlin.utils.DFS
import java.io.ByteArrayOutputStream
import java.io.Closeable
import java.io.File
import java.io.PrintStream
import java.io.*
import java.nio.charset.Charset
import java.util.regex.Pattern
@@ -386,6 +384,7 @@ abstract class BasicBoxTest(
}
processJsProgram(translationResult.program, units.filterIsInstance<TranslationUnit.SourceFile>().map { it.file })
checkSourceMap(outputFile, translationResult.program)
}
protected fun processJsProgram(program: JsProgram, psiFiles: List<KtFile>) {
@@ -395,6 +394,56 @@ abstract class BasicBoxTest(
program.verifyAst()
}
private fun checkSourceMap(outputFile: File, program: JsProgram) {
val generatedProgram = JsProgram()
generatedProgram.globalBlock.statements += program.globalBlock.statements.map { it.deepCopy() }
generatedProgram.accept(object : RecursiveJsVisitor() {
override fun visitObjectLiteral(x: JsObjectLiteral) {
super.visitObjectLiteral(x)
x.isMultiline = false
}
override fun visitVars(x: JsVars) {
x.isMultiline = false
super.visitVars(x)
}
})
removeLocationFromBlocks(generatedProgram)
generatedProgram.accept(AmbiguousAstSourcePropagation())
val output = TextOutputImpl()
val sourceMapBuilder = SourceMap3Builder(outputFile, output, SourceMapBuilderConsumer())
generatedProgram.accept(JsSourceGenerationVisitor(output, sourceMapBuilder))
val code = output.toString()
val generatedSourceMap = sourceMapBuilder.build()
val codeWithLines = generatedProgram.toStringWithLineNumbers()
val parsedProgram = JsProgram()
parsedProgram.globalBlock.statements += parse(code, ThrowExceptionOnErrorReporter, parsedProgram.scope, outputFile.path)
removeLocationFromBlocks(parsedProgram)
val sourceMapParseResult = SourceMapParser.parse(StringReader(generatedSourceMap))
val sourceMap = when (sourceMapParseResult) {
is SourceMapSuccess -> sourceMapParseResult.value
is SourceMapError -> error("Could not parse source map: ${sourceMapParseResult.message}")
}
val remapper = SourceMapLocationRemapper(mapOf(outputFile.path to sourceMap))
remapper.remap(parsedProgram)
val codeWithRemappedLines = parsedProgram.toStringWithLineNumbers()
TestCase.assertEquals(codeWithLines, codeWithRemappedLines)
}
private fun removeLocationFromBlocks(program: JsProgram) {
program.globalBlock.accept(object : RecursiveJsVisitor() {
override fun visitBlock(x: JsBlock) {
super.visitBlock(x)
x.source = null
}
})
}
private fun createPsiFile(fileName: String): KtFile {
val psiManager = PsiManager.getInstance(project)
val fileSystem = VirtualFileManager.getInstance().getFileSystem(StandardFileSystems.FILE_PROTOCOL)
@@ -422,12 +471,10 @@ abstract class BasicBoxTest(
configuration.put(JSConfigurationKeys.MODULE_KIND, module.moduleKind)
configuration.put(JSConfigurationKeys.TARGET, EcmaVersion.v5)
configuration.put(JSConfigurationKeys.SOURCE_MAP, generateSourceMap)
val hasFilesToRecompile = module.hasFilesToRecompile
configuration.put(JSConfigurationKeys.META_INFO, multiModule)
configuration.put(JSConfigurationKeys.SERIALIZE_FRAGMENTS, hasFilesToRecompile)
configuration.put(JSConfigurationKeys.SOURCE_MAP, hasFilesToRecompile)
configuration.put(JSConfigurationKeys.SOURCE_MAP, hasFilesToRecompile || generateSourceMap)
if (additionalMetadata != null) {
val metadata = PackagesWithHeaderMetadata(
js/js.tests/test/org/jetbrains/kotlin/js/test/utils/LineCollector.kt
+15 -5
View File
@@ -31,11 +31,21 @@ class LineCollector : RecursiveJsVisitor() {
}
private fun handleNodeLocation(node: JsNode) {
(node.source as? PsiElement)?.let { source ->
val file = source.containingFile
val offset = source.node.startOffset
val document = file.viewProvider.document!!
val line = document.getLineNumber(offset)
val source = node.source
val line = when (source) {
is PsiElement -> {
val file = source.containingFile
val offset = source.node.startOffset
val document = file.viewProvider.document!!
document.getLineNumber(offset)
}
is JsLocation -> {
source.startLine
}
else -> null
}
if (line != null) {
currentStatement?.let {
val linesByStatement = lineNumbersByStatement.getOrPut(it, ::mutableListOf)
if (linesByStatement.lastOrNull() != line) {
js/js.tests/test/org/jetbrains/kotlin/js/test/utils/lineNumberUtils.kt
+27
View File
@@ -0,0 +1,27 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.js.test.utils
import org.jetbrains.kotlin.js.backend.ast.JsProgram
import org.jetbrains.kotlin.js.util.TextOutputImpl
fun JsProgram.toStringWithLineNumbers(): String {
val output = TextOutputImpl()
val lineCollector = LineCollector().also { it.accept(this) }
LineOutputToStringVisitor(output, lineCollector).accept(this.globalBlock)
return output.toString()
}
js/js.translator/src/org/jetbrains/kotlin/js/facade/SourceMapBuilderConsumer.java
+1 -1
View File
@@ -23,7 +23,7 @@ import com.intellij.util.PairConsumer;
import org.jetbrains.kotlin.js.backend.ast.JsLocation;
import org.jetbrains.kotlin.js.sourceMap.SourceMapBuilder;
class SourceMapBuilderConsumer implements PairConsumer<SourceMapBuilder, Object> {
public class SourceMapBuilderConsumer implements PairConsumer<SourceMapBuilder, Object> {
@Override
public void consume(SourceMapBuilder builder, Object sourceInfo) {
if (sourceInfo instanceof PsiElement) {
js/js.translator/src/org/jetbrains/kotlin/js/sourceMap/JsSourceGenerationVisitor.java
+1 -8
View File
@@ -57,9 +57,7 @@ public class JsSourceGenerationVisitor extends JsToStringGenerationVisitor imple
@Override
public void accept(JsNode node) {
if (!(node instanceof JsNameRef) && !(node instanceof JsThisRef)) {
mapSource(node);
}
mapSource(node);
super.accept(node);
}
@@ -77,11 +75,6 @@ public class JsSourceGenerationVisitor extends JsToStringGenerationVisitor imple
}
}
@Override
protected void beforeNodePrinted(JsNode node) {
mapSource(node);
}
@Override
public void visitProgram(@NotNull JsProgram program) {
program.acceptChildren(this);