Remap source maps in JS DCE. Improve JS DCE error logging

See KT-19821
This commit is contained in:
Alexey Andreev
2017-08-25 15:38:01 +03:00
parent 1350e3c4ac
commit c66bc0b0e9
28 changed files with 196 additions and 42 deletions
@@ -22,10 +22,7 @@ import org.jetbrains.kotlin.cli.common.arguments.K2JSDceArguments
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.js.dce.DeadCodeElimination
import org.jetbrains.kotlin.js.dce.InputFile
import org.jetbrains.kotlin.js.dce.extractRoots
import org.jetbrains.kotlin.js.dce.printTree
import org.jetbrains.kotlin.js.dce.*
import java.io.File
class K2JSDce : CLITool<K2JSDceArguments>() {
@@ -38,7 +35,9 @@ class K2JSDce : CLITool<K2JSDceArguments>() {
val inputName = parts[0]
val moduleName = parts.getOrNull(1) ?: ""
val resolvedModuleName = if (!moduleName.isEmpty()) moduleName else File(inputName).nameWithoutExtension
InputFile(inputName, File(baseDir, resolvedModuleName + ".js").absolutePath, resolvedModuleName)
val pathToSourceMapCandidate = "$inputName.map"
val pathToSourceMap = if (File(pathToSourceMapCandidate).exists()) pathToSourceMapCandidate else null
InputFile(inputName, pathToSourceMap, File(baseDir, resolvedModuleName + ".js").absolutePath, resolvedModuleName)
}
if (files.isEmpty() && !arguments.version) {
@@ -51,9 +50,16 @@ class K2JSDce : CLITool<K2JSDceArguments>() {
val includedDeclarations = arguments.declarationsToKeep.orEmpty().toSet()
val dceResult = DeadCodeElimination.run(files, includedDeclarations) {
messageCollector.report(CompilerMessageSeverity.LOGGING, it)
val logConsumer = { level: DCELogLevel, message: String ->
val severity = when (level) {
DCELogLevel.ERROR -> CompilerMessageSeverity.ERROR
DCELogLevel.WARN -> CompilerMessageSeverity.WARNING
DCELogLevel.INFO -> CompilerMessageSeverity.LOGGING
}
messageCollector.report(severity, message)
}
val dceResult = DeadCodeElimination.run(files, includedDeclarations, logConsumer)
if (dceResult.status == DeadCodeEliminationStatus.FAILED) return ExitCode.COMPILATION_ERROR
val nodes = dceResult.reachableNodes
val reachabilitySeverity = if (arguments.printReachabilityInfo) CompilerMessageSeverity.INFO else CompilerMessageSeverity.LOGGING
+3
View File
@@ -0,0 +1,3 @@
$TESTDATA_DIR$/parseError.js
-output-dir
$TEMP_DIR$/min
+1
View File
@@ -0,0 +1 @@
..
+2
View File
@@ -0,0 +1,2 @@
error: at compiler/testData/cli/js-dce/parseError.js (1, 1): syntax error
COMPILATION_ERROR
+1
View File
@@ -0,0 +1 @@
// ABSENT: min/parseError.js
+3
View File
@@ -0,0 +1,3 @@
$TESTDATA_DIR$/withSourceMap.js
-output-dir
$TEMP_DIR$/min
+24
View File
@@ -0,0 +1,24 @@
if (typeof kotlin === 'undefined') {
throw new Error("Error loading module 'sample'. Its dependency 'kotlin' was not found. Please, check whether 'kotlin' is loaded prior to 'sample'.");
}
var sample = function (_, Kotlin) {
'use strict';
var println = Kotlin.kotlin.io.println_s8jyv4$;
function foo() {
println('foo');
}
function bar() {
println('bar');
}
function main(args) {
foo();
}
_.foo = foo;
_.bar = bar;
_.main_kand9s$ = main;
main([]);
Kotlin.defineModule('sample', _);
return _;
}(typeof sample === 'undefined' ? {} : sample, kotlin);
//# sourceMappingURL=sample.js.map
+1
View File
@@ -0,0 +1 @@
{"version":3,"file":"sample.js","sources":["sample.kt"],"sourcesContent":[null],"names":[],"mappings":";;;;;;;IACI,QAAQ,KAAR,C;EACJ,C;;IAGI,QAAQ,KAAR,C;EACJ,C;;IAGI,K;EACJ,C;;;;;;;;"}
+1
View File
@@ -0,0 +1 @@
OK
+2
View File
@@ -0,0 +1,2 @@
// EXISTS: min/withSourceMap.js
// EXISTS: min/withSourceMap.js.map
@@ -674,6 +674,12 @@ public class CliTestGenerated extends AbstractCliTest {
doJsDceTest(fileName);
}
@TestMetadata("parseError.args")
public void testParseError() throws Exception {
String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/cli/js-dce/parseError.args");
doJsDceTest(fileName);
}
@TestMetadata("printReachability.args")
public void testPrintReachability() throws Exception {
String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/cli/js-dce/printReachability.args");
@@ -685,5 +691,11 @@ public class CliTestGenerated extends AbstractCliTest {
String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/cli/js-dce/simple.args");
doJsDceTest(fileName);
}
@TestMetadata("withSourceMap.args")
public void testWithSourceMap() throws Exception {
String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/cli/js-dce/withSourceMap.args");
doJsDceTest(fileName);
}
}
}
@@ -0,0 +1,23 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.js.dce
enum class DCELogLevel {
INFO,
WARN,
ERROR
}
@@ -18,14 +18,25 @@ package org.jetbrains.kotlin.js.dce
import com.google.gwt.dev.js.rhino.CodePosition
import com.google.gwt.dev.js.rhino.ErrorReporter
import org.jetbrains.kotlin.js.backend.JsToStringGenerationVisitor
import org.jetbrains.kotlin.js.backend.ast.*
import org.jetbrains.kotlin.js.dce.Context.Node
import org.jetbrains.kotlin.js.facade.SourceMapBuilderConsumer
import org.jetbrains.kotlin.js.inline.util.collectDefinedNames
import org.jetbrains.kotlin.js.inline.util.fixForwardNameReferences
import org.jetbrains.kotlin.js.parser.parse
import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapError
import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapLocationRemapper
import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapParser
import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapSuccess
import org.jetbrains.kotlin.js.sourceMap.SourceFilePathResolver
import org.jetbrains.kotlin.js.sourceMap.SourceMap3Builder
import org.jetbrains.kotlin.js.util.TextOutputImpl
import java.io.File
import java.io.FileInputStream
import java.io.InputStreamReader
class DeadCodeElimination(val logConsumer: (String) -> Unit) {
class DeadCodeElimination(private val logConsumer: (DCELogLevel, String) -> Unit) {
val moduleMapping = mutableMapOf<JsBlock, String>()
private val reachableNames = mutableSetOf<String>()
@@ -63,18 +74,41 @@ class DeadCodeElimination(val logConsumer: (String) -> Unit) {
fun run(
inputFiles: Collection<InputFile>,
rootReachableNames: Set<String>,
logConsumer: (String) -> Unit
logConsumer: (DCELogLevel, String) -> Unit
): DeadCodeEliminationResult {
val program = JsProgram()
val dce = DeadCodeElimination(logConsumer)
var hasErrors = false
val blocks = inputFiles.map { file ->
val block = JsGlobalBlock()
val code = File(file.path).readText()
block.statements += parse(code, reporter, program.scope, file.path)
val statements = parse(code, Reporter(file.path, logConsumer), program.scope, file.path) ?: run {
hasErrors = true
return@map block
}
val sourceMapParse = file.pathToSourceMap
?.let { InputStreamReader(FileInputStream(it), "UTF-8") }
?.use { SourceMapParser.parse(it) }
when (sourceMapParse) {
is SourceMapError -> {
logConsumer(
DCELogLevel.WARN,
"Error parsing source map file ${file.pathToSourceMap}: ${sourceMapParse.message}")
}
is SourceMapSuccess -> {
val sourceMap = sourceMapParse.value
val remapper = SourceMapLocationRemapper(sourceMap)
statements.forEach { remapper.remap(it) }
}
}
block.statements += statements
file.moduleName?.let { dce.moduleMapping[block] = it }
block
}
if (hasErrors) return DeadCodeEliminationResult(emptySet(), DeadCodeEliminationStatus.FAILED)
program.globalBlock.statements += blocks
program.globalBlock.fixForwardNameReferences()
@@ -82,22 +116,34 @@ class DeadCodeElimination(val logConsumer: (String) -> Unit) {
dce.apply(program.globalBlock)
for ((file, block) in inputFiles.zip(blocks)) {
val sourceMapFile = File(file.outputPath + ".map")
val textOutput = TextOutputImpl()
val sourceMapBuilder = SourceMap3Builder(File(file.outputPath), textOutput, "")
val consumer = SourceMapBuilderConsumer(sourceMapBuilder, SourceFilePathResolver(mutableListOf()), true, true)
block.accept(JsToStringGenerationVisitor(textOutput, consumer))
val sourceMapContent = sourceMapBuilder.build()
sourceMapBuilder.addLink()
with(File(file.outputPath)) {
parentFile.mkdirs()
writeText(block.toString())
writeText(textOutput.toString())
}
if (file.pathToSourceMap != null) {
sourceMapFile.writeText(sourceMapContent)
}
}
return DeadCodeEliminationResult(dce.reachableNodes)
return DeadCodeEliminationResult(dce.reachableNodes, DeadCodeEliminationStatus.OK)
}
private val reporter = object : ErrorReporter {
private class Reporter(private val fileName: String, private val logConsumer: (DCELogLevel, String) -> Unit) : ErrorReporter {
override fun warning(message: String, startPosition: CodePosition, endPosition: CodePosition) {
println("[WARN] at ${startPosition.line}, ${startPosition.offset}: $message")
logConsumer(DCELogLevel.WARN, "at $fileName (${startPosition.line + 1}, ${startPosition.offset + 1}): $message")
}
override fun error(message: String, startPosition: CodePosition, endPosition: CodePosition) {
println("[ERRO] at ${startPosition.line}, ${startPosition.offset}: $message")
logConsumer(DCELogLevel.ERROR, "at $fileName (${startPosition.line + 1}, ${startPosition.offset + 1}): $message")
}
}
}
@@ -16,4 +16,4 @@
package org.jetbrains.kotlin.js.dce
class DeadCodeEliminationResult(val reachableNodes: Set<Context.Node>)
class DeadCodeEliminationResult(val reachableNodes: Set<Context.Node>, val status: DeadCodeEliminationStatus)
@@ -0,0 +1,22 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.js.dce
enum class DeadCodeEliminationStatus {
OK,
FAILED
}
@@ -16,4 +16,4 @@
package org.jetbrains.kotlin.js.dce
class InputFile(val path: String, val outputPath: String, val moduleName: String? = null)
class InputFile(val path: String, val pathToSourceMap: String?, val outputPath: String, val moduleName: String? = null)
@@ -23,7 +23,7 @@ import org.jetbrains.kotlin.js.inline.util.collectLocalVariables
class ReachabilityTracker(
private val context: Context,
private val analysisResult: AnalysisResult,
private val logConsumer: (String) -> Unit
private val logConsumer: (DCELogLevel, String) -> Unit
) : RecursiveJsVisitor() {
companion object {
private val CALL_FUNCTIONS = setOf("call", "apply")
@@ -217,7 +217,7 @@ class ReachabilityTracker(
}
private fun report(message: String) {
logConsumer(" ".repeat(depth) + message)
logConsumer(DCELogLevel.INFO, " ".repeat(depth) + message)
}
private fun reportAndNest(message: String, dueTo: JsNode?, action: () -> Unit) {
@@ -92,7 +92,7 @@ class JsCallChecker(
val parserScope = JsFunctionScope(JsRootScope(JsProgram()), "<js fun>")
val statements = parse(code, errorReporter, parserScope, reportOn.containingFile?.name ?: "<unknown file>")
if (statements.isEmpty()) {
if (statements == null || statements.isEmpty()) {
context.trace.report(ErrorsJs.JSCODE_NO_JAVASCRIPT_PRODUCED.on(argument))
}
} catch (e: AbortParsingException) {
@@ -209,7 +209,8 @@ class FunctionReader(
}
val position = info.offsetToSourceMapping[offset]
val functionExpr = parseFunction(source, info.filePath, position, offset, ThrowExceptionOnErrorReporter, JsRootScope(JsProgram()))
val functionExpr = parseFunction(source, info.filePath, position, offset, ThrowExceptionOnErrorReporter, JsRootScope(JsProgram())) ?:
return null
functionExpr.fixForwardNameReferences()
val (function, wrapper) = if (isWrapped) {
InlineMetadata.decomposeWrapper(functionExpr) ?: return null
@@ -22,20 +22,20 @@ import org.jetbrains.kotlin.js.backend.ast.*
import java.io.Reader
import java.io.StringReader
fun parse(code: String, reporter: ErrorReporter, scope: JsScope, fileName: String): List<JsStatement> {
fun parse(code: String, reporter: ErrorReporter, scope: JsScope, fileName: String): List<JsStatement>? {
val insideFunction = scope is JsFunctionScope
val node = parse(code, CodePosition(0, 0), 0, reporter, insideFunction, Parser::parse)
return node.toJsAst(scope, fileName) {
return node?.toJsAst(scope, fileName) {
mapStatements(it)
}
}
fun parseFunction(code: String, fileName: String, position: CodePosition, offset: Int, reporter: ErrorReporter, scope: JsScope): JsFunction {
fun parseFunction(code: String, fileName: String, position: CodePosition, offset: Int, reporter: ErrorReporter, scope: JsScope): JsFunction? {
val rootNode = parse(code, position, offset, reporter, insideFunction = false) {
addListener(FunctionParsingObserver())
primaryExpr(it)
}
return rootNode.toJsAst(scope, fileName, JsAstMapper::mapFunction)
return rootNode?.toJsAst(scope, fileName, JsAstMapper::mapFunction)
}
private class FunctionParsingObserver : ParserListener {
@@ -60,13 +60,13 @@ private fun parse(
reporter: ErrorReporter,
insideFunction: Boolean,
parseAction: Parser.(TokenStream)->Any
): Node {
): Node? {
Context.enter().errorReporter = reporter
try {
val ts = TokenStream(StringReader(code, offset), "<parser>", startPosition)
val parser = Parser(IRFactory(ts), insideFunction)
return parser.parseAction(ts) as Node
return parser.parseAction(ts) as? Node
} finally {
Context.exit()
}
@@ -25,8 +25,8 @@ abstract class AbstractDceTest : TestCase() {
fun doTest(filePath: String) {
val file = File(filePath)
val fileContents = file.readText()
val inputFile = InputFile(filePath, File(pathToOutputDir, file.relativeTo(File(pathToTestDir)).path).path, "main")
val dceResult = DeadCodeElimination.run(setOf(inputFile), extractDeclarations(REQUEST_REACHABLE_PATTERN, fileContents)) {}
val inputFile = InputFile(filePath, null, File(pathToOutputDir, file.relativeTo(File(pathToTestDir)).path).path, "main")
val dceResult = DeadCodeElimination.run(setOf(inputFile), extractDeclarations(REQUEST_REACHABLE_PATTERN, fileContents)) { _, _ -> }
val reachableNodeStrings = dceResult.reachableNodes.map { it.toString().removePrefix("<unknown>.") }.toSet()
for (assertedDeclaration in extractDeclarations(ASSERT_REACHABLE_PATTERN, fileContents)) {
@@ -440,7 +440,7 @@ abstract class BasicBoxTest(
val codeWithLines = generatedProgram.toStringWithLineNumbers()
val parsedProgram = JsProgram()
parsedProgram.globalBlock.statements += parse(code, ThrowExceptionOnErrorReporter, parsedProgram.scope, outputFile.path)
parsedProgram.globalBlock.statements += parse(code, ThrowExceptionOnErrorReporter, parsedProgram.scope, outputFile.path).orEmpty()
removeLocationFromBlocks(parsedProgram)
val sourceMapParseResult = SourceMapParser.parse(StringReader(generatedSourceMap))
val sourceMap = when (sourceMapParseResult) {
@@ -524,12 +524,12 @@ abstract class BasicBoxTest(
val kotlinJsLibOutput = File(workDir, "kotlin.min.js").path
val kotlinTestJsLibOutput = File(workDir, "kotlin-test.min.js").path
val kotlinJsInputFile = InputFile(kotlinJsLib, kotlinJsLibOutput, "kotlin")
val kotlinTestJsInputFile = InputFile(kotlinTestJsLib, kotlinTestJsLibOutput, "kotlin-test")
val kotlinJsInputFile = InputFile(kotlinJsLib, null, kotlinJsLibOutput, "kotlin")
val kotlinTestJsInputFile = InputFile(kotlinTestJsLib, null, kotlinTestJsLibOutput, "kotlin-test")
val filesToMinify = generatedJsFiles.associate { (fileName, module) ->
val inputFileName = File(fileName).nameWithoutExtension
fileName to InputFile(fileName, File(workDir, inputFileName + ".min.js").absolutePath, module.name)
fileName to InputFile(fileName, null, File(workDir, inputFileName + ".min.js").absolutePath, module.name)
}
val testFunctionFqn = testModuleName + (if (testPackage.isNullOrEmpty()) "" else ".$testPackage") + ".$testFunction"
@@ -538,7 +538,7 @@ abstract class BasicBoxTest(
"kotlin.kotlin.io.output.buffer", "kotlin-test.kotlin.test.overrideAsserter_wbnzx$"
)
val allFilesToMinify = filesToMinify.values + kotlinJsInputFile + kotlinTestJsInputFile
val dceResult = DeadCodeElimination.run(allFilesToMinify, additionalReachableNodes) { }
val dceResult = DeadCodeElimination.run(allFilesToMinify, additionalReachableNodes) { _, _ -> }
val reachableNodes = dceResult.reachableNodes
minificationThresholdChecker(reachableNodes.size)
@@ -57,8 +57,8 @@ class NameResolutionTest {
val expectedCode = FileUtil.loadFile(File(expectedName))
val parserScope = JsFunctionScope(JsRootScope(JsProgram()), "<js fun>")
val originalAst = JsGlobalBlock().apply { statements += parse(originalCode, errorReporter, parserScope, originalName) }
val expectedAst = JsGlobalBlock().apply { statements += parse(expectedCode, errorReporter, parserScope, expectedName) }
val originalAst = JsGlobalBlock().apply { statements += parse(originalCode, errorReporter, parserScope, originalName).orEmpty() }
val expectedAst = JsGlobalBlock().apply { statements += parse(expectedCode, errorReporter, parserScope, expectedName).orEmpty() }
originalAst.accept(object : RecursiveJsVisitor() {
val cache = mutableMapOf<JsName, JsName>()
@@ -54,7 +54,7 @@ abstract class BasicOptimizerTest(private var basePath: String) {
private fun checkOptimizer(unoptimizedCode: String, optimizedCode: String) {
val parserScope = JsFunctionScope(JsRootScope(JsProgram()), "<js fun>")
val unoptimizedAst = parse(unoptimizedCode, errorReporter, parserScope, "<unknown file>")
val unoptimizedAst = parse(unoptimizedCode, errorReporter, parserScope, "<unknown file>")!!
updateMetadata(unoptimizedCode, unoptimizedAst)
@@ -62,7 +62,7 @@ abstract class BasicOptimizerTest(private var basePath: String) {
process(statement)
}
val optimizedAst = parse(optimizedCode, errorReporter, parserScope, "<unknown file>")
val optimizedAst = parse(optimizedCode, errorReporter, parserScope, "<unknown file>")!!
Assert.assertEquals(astToString(optimizedAst), astToString(unoptimizedAst))
}
@@ -225,6 +225,9 @@ class Kotlin2JsGradlePluginIT : BaseGradleIT() {
project.build("runRhino") {
println(output)
assertSuccessful()
val pathPrefix = "mainProject/build/min"
assertFileExists("$pathPrefix/exampleapp.js.map")
assertFileExists("$pathPrefix/examplelib.js.map")
}
}
@@ -18,10 +18,12 @@ dependencies {
}
compileKotlin2Js.kotlinOptions.outputFile = "${buildDir}/examplelib.js"
compileKotlin2Js.kotlinOptions.sourceMap = true
jar {
from buildDir
include "**/*.js"
include "**/*.js.map"
}
jar.dependsOn(compileKotlin2Js)
@@ -68,8 +68,8 @@ class KotlinJsDcePlugin : Plugin<Project> {
val zippedFiles = UnionFileCollection(configuration.map { project.zipTree(it) })
val files = project.fileTree(tmpDir)
.filter { it.path.endsWith(".js") }
.filter { File(it.path.removeSuffix(".js") + ".meta.js").exists() }
.filter { file -> SUFFIXES.any { file.path.endsWith(it) } }
.filter { file -> SUFFIXES.any { File(file.path.removeSuffix(it) + ".meta.js").exists() } }
// This intermediate task is needed due to bug in Gradle that causes infinite loops in continuous build mode
val unpackName = sourceSet.getTaskName(UNPACK_DEPENDENCIES_TASK_PREFIX, TASK_SUFFIX)
@@ -93,5 +93,6 @@ class KotlinJsDcePlugin : Plugin<Project> {
private const val UNPACK_DEPENDENCIES_TASK_PREFIX = "unpackDependencies"
private const val DEPENDENCIES_TASK_PREFIX = "copyDependencies"
private const val DCE_TASK_PREFIX = "runDce"
private val SUFFIXES = listOf(".js", ".js.map")
}
}
@@ -43,14 +43,14 @@ fun main(args: Array<String>) {
fun File.relativizeIfNecessary(): String = baseDir.relativize(canonicalFile.toPath()).toString()
val wrapperFile = File(args[2])
val wrapper = parse(wrapperFile.readText(), ThrowExceptionOnErrorReporter, program.scope, wrapperFile.relativizeIfNecessary())
val wrapper = parse(wrapperFile.readText(), ThrowExceptionOnErrorReporter, program.scope, wrapperFile.relativizeIfNecessary())!!
val insertionPlace = wrapper.createInsertionPlace()
val allFiles = mutableListOf<File>()
args.drop(3).map { File(it) }.forEach { collectFiles(it, allFiles) }
for (file in allFiles) {
val statements = parse(file.readText(), ThrowExceptionOnErrorReporter, program.scope, file.relativizeIfNecessary())
val statements = parse(file.readText(), ThrowExceptionOnErrorReporter, program.scope, file.relativizeIfNecessary())!!
val block = JsBlock(statements)
block.fixForwardNameReferences()