Remap source maps in JS DCE. Improve JS DCE error logging

See KT-19821

js/js.dce/src/org/jetbrains/kotlin/js/dce/DCELogLevel.kt

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.js.dce
+
+enum class DCELogLevel {
+    INFO,
+    WARN,
+    ERROR
+}

js/js.dce/src/org/jetbrains/kotlin/js/dce/DeadCodeElimination.kt

@@ -18,14 +18,25 @@ package org.jetbrains.kotlin.js.dce
 
 import com.google.gwt.dev.js.rhino.CodePosition
 import com.google.gwt.dev.js.rhino.ErrorReporter
+import org.jetbrains.kotlin.js.backend.JsToStringGenerationVisitor
 import org.jetbrains.kotlin.js.backend.ast.*
 import org.jetbrains.kotlin.js.dce.Context.Node
+import org.jetbrains.kotlin.js.facade.SourceMapBuilderConsumer
 import org.jetbrains.kotlin.js.inline.util.collectDefinedNames
 import org.jetbrains.kotlin.js.inline.util.fixForwardNameReferences
 import org.jetbrains.kotlin.js.parser.parse
+import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapError
+import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapLocationRemapper
+import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapParser
+import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapSuccess
+import org.jetbrains.kotlin.js.sourceMap.SourceFilePathResolver
+import org.jetbrains.kotlin.js.sourceMap.SourceMap3Builder
+import org.jetbrains.kotlin.js.util.TextOutputImpl
 import java.io.File
+import java.io.FileInputStream
+import java.io.InputStreamReader
 
-class DeadCodeElimination(val logConsumer: (String) -> Unit) {
+class DeadCodeElimination(private val logConsumer: (DCELogLevel, String) -> Unit) {
     val moduleMapping = mutableMapOf<JsBlock, String>()
     private val reachableNames = mutableSetOf<String>()
 
@@ -63,18 +74,41 @@ class DeadCodeElimination(val logConsumer: (String) -> Unit) {
         fun run(
                 inputFiles: Collection<InputFile>,
                 rootReachableNames: Set<String>,
-                logConsumer: (String) -> Unit
+                logConsumer: (DCELogLevel, String) -> Unit
         ): DeadCodeEliminationResult {
             val program = JsProgram()
             val dce = DeadCodeElimination(logConsumer)
 
+            var hasErrors = false
             val blocks = inputFiles.map { file ->
                 val block = JsGlobalBlock()
                 val code = File(file.path).readText()
-                block.statements += parse(code, reporter, program.scope, file.path)
+                val statements = parse(code, Reporter(file.path, logConsumer), program.scope, file.path) ?: run {
+                    hasErrors = true
+                    return@map block
+                }
+                val sourceMapParse = file.pathToSourceMap
+                        ?.let { InputStreamReader(FileInputStream(it), "UTF-8") }
+                        ?.use { SourceMapParser.parse(it) }
+                when (sourceMapParse) {
+                    is SourceMapError -> {
+                        logConsumer(
+                                DCELogLevel.WARN,
+                                "Error parsing source map file ${file.pathToSourceMap}: ${sourceMapParse.message}")
+                    }
+                    is SourceMapSuccess -> {
+                        val sourceMap = sourceMapParse.value
+                        val remapper = SourceMapLocationRemapper(sourceMap)
+                        statements.forEach { remapper.remap(it) }
+                    }
+                }
+                block.statements += statements
                 file.moduleName?.let { dce.moduleMapping[block] = it }
                 block
             }
+
+            if (hasErrors) return DeadCodeEliminationResult(emptySet(), DeadCodeEliminationStatus.FAILED)
+
             program.globalBlock.statements += blocks
             program.globalBlock.fixForwardNameReferences()
 
@@ -82,22 +116,34 @@ class DeadCodeElimination(val logConsumer: (String) -> Unit) {
             dce.apply(program.globalBlock)
 
             for ((file, block) in inputFiles.zip(blocks)) {
+                val sourceMapFile = File(file.outputPath + ".map")
+                val textOutput = TextOutputImpl()
+                val sourceMapBuilder = SourceMap3Builder(File(file.outputPath), textOutput, "")
+                val consumer = SourceMapBuilderConsumer(sourceMapBuilder, SourceFilePathResolver(mutableListOf()), true, true)
+                block.accept(JsToStringGenerationVisitor(textOutput, consumer))
+                val sourceMapContent = sourceMapBuilder.build()
+                sourceMapBuilder.addLink()
+
                 with(File(file.outputPath)) {
                     parentFile.mkdirs()
-                    writeText(block.toString())
+                    writeText(textOutput.toString())
+                }
+
+                if (file.pathToSourceMap != null) {
+                    sourceMapFile.writeText(sourceMapContent)
                 }
             }
 
-            return DeadCodeEliminationResult(dce.reachableNodes)
+            return DeadCodeEliminationResult(dce.reachableNodes, DeadCodeEliminationStatus.OK)
         }
 
-        private val reporter = object : ErrorReporter {
+        private class Reporter(private val fileName: String, private val logConsumer: (DCELogLevel, String) -> Unit) : ErrorReporter {
             override fun warning(message: String, startPosition: CodePosition, endPosition: CodePosition) {
-                println("[WARN] at ${startPosition.line}, ${startPosition.offset}: $message")
+                logConsumer(DCELogLevel.WARN, "at $fileName (${startPosition.line + 1}, ${startPosition.offset + 1}): $message")
             }
 
             override fun error(message: String, startPosition: CodePosition, endPosition: CodePosition) {
-                println("[ERRO] at ${startPosition.line}, ${startPosition.offset}: $message")
+                logConsumer(DCELogLevel.ERROR, "at $fileName (${startPosition.line + 1}, ${startPosition.offset + 1}): $message")
             }
         }
     }

js/js.dce/src/org/jetbrains/kotlin/js/dce/DeadCodeEliminationResult.kt

@@ -16,4 +16,4 @@
 
 package org.jetbrains.kotlin.js.dce
 
-class DeadCodeEliminationResult(val reachableNodes: Set<Context.Node>)
+class DeadCodeEliminationResult(val reachableNodes: Set<Context.Node>, val status: DeadCodeEliminationStatus)

js/js.dce/src/org/jetbrains/kotlin/js/dce/DeadCodeEliminationStatus.kt

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.js.dce
+
+enum class DeadCodeEliminationStatus {
+    OK,
+    FAILED
+}

js/js.dce/src/org/jetbrains/kotlin/js/dce/InputFile.kt

@@ -16,4 +16,4 @@
 
 package org.jetbrains.kotlin.js.dce
 
-class InputFile(val path: String, val outputPath: String, val moduleName: String? = null)
+class InputFile(val path: String, val pathToSourceMap: String?, val outputPath: String, val moduleName: String? = null)

js/js.dce/src/org/jetbrains/kotlin/js/dce/ReachabilityTracker.kt

@@ -23,7 +23,7 @@ import org.jetbrains.kotlin.js.inline.util.collectLocalVariables
 class ReachabilityTracker(
         private val context: Context,
         private val analysisResult: AnalysisResult,
-        private val logConsumer: (String) -> Unit
+        private val logConsumer: (DCELogLevel, String) -> Unit
 ) : RecursiveJsVisitor() {
     companion object {
         private val CALL_FUNCTIONS = setOf("call", "apply")
@@ -217,7 +217,7 @@ class ReachabilityTracker(
     }
 
     private fun report(message: String) {
-        logConsumer("  ".repeat(depth) + message)
+        logConsumer(DCELogLevel.INFO, "  ".repeat(depth) + message)
     }
 
     private fun reportAndNest(message: String, dueTo: JsNode?, action: () -> Unit) {

js/js.frontend/src/org/jetbrains/kotlin/js/resolve/diagnostics/JsCallChecker.kt

@@ -92,7 +92,7 @@ class JsCallChecker(
             val parserScope = JsFunctionScope(JsRootScope(JsProgram()), "<js fun>")
             val statements = parse(code, errorReporter, parserScope, reportOn.containingFile?.name ?: "<unknown file>")
 
-            if (statements.isEmpty()) {
+            if (statements == null || statements.isEmpty()) {
                 context.trace.report(ErrorsJs.JSCODE_NO_JAVASCRIPT_PRODUCED.on(argument))
             }
         } catch (e: AbortParsingException) {

js/js.inliner/src/org/jetbrains/kotlin/js/inline/FunctionReader.kt

@@ -209,7 +209,8 @@ class FunctionReader(
         }
 
         val position = info.offsetToSourceMapping[offset]
-        val functionExpr = parseFunction(source, info.filePath, position, offset, ThrowExceptionOnErrorReporter, JsRootScope(JsProgram()))
+        val functionExpr = parseFunction(source, info.filePath, position, offset, ThrowExceptionOnErrorReporter, JsRootScope(JsProgram())) ?:
+                           return null
         functionExpr.fixForwardNameReferences()
         val (function, wrapper) = if (isWrapped) {
             InlineMetadata.decomposeWrapper(functionExpr) ?: return null

js/js.parser/src/org/jetbrains/kotlin/js/parser/parserUtils.kt

@@ -22,20 +22,20 @@ import org.jetbrains.kotlin.js.backend.ast.*
 import java.io.Reader
 import java.io.StringReader
 
-fun parse(code: String, reporter: ErrorReporter, scope: JsScope, fileName: String): List<JsStatement> {
+fun parse(code: String, reporter: ErrorReporter, scope: JsScope, fileName: String): List<JsStatement>? {
     val insideFunction = scope is JsFunctionScope
     val node = parse(code, CodePosition(0, 0), 0, reporter, insideFunction, Parser::parse)
-    return node.toJsAst(scope, fileName) {
+    return node?.toJsAst(scope, fileName) {
         mapStatements(it)
     }
 }
 
-fun parseFunction(code: String, fileName: String, position: CodePosition, offset: Int, reporter: ErrorReporter, scope: JsScope): JsFunction {
+fun parseFunction(code: String, fileName: String, position: CodePosition, offset: Int, reporter: ErrorReporter, scope: JsScope): JsFunction? {
     val rootNode = parse(code, position, offset, reporter, insideFunction = false) {
         addListener(FunctionParsingObserver())
         primaryExpr(it)
     }
-    return rootNode.toJsAst(scope, fileName, JsAstMapper::mapFunction)
+    return rootNode?.toJsAst(scope, fileName, JsAstMapper::mapFunction)
 }
 
 private class FunctionParsingObserver : ParserListener {
@@ -60,13 +60,13 @@ private fun parse(
         reporter: ErrorReporter,
         insideFunction: Boolean,
         parseAction: Parser.(TokenStream)->Any
-): Node {
+): Node? {
     Context.enter().errorReporter = reporter
 
     try {
         val ts = TokenStream(StringReader(code, offset), "<parser>", startPosition)
         val parser = Parser(IRFactory(ts), insideFunction)
-        return parser.parseAction(ts) as Node
+        return parser.parseAction(ts) as? Node
     } finally {
         Context.exit()
     }

js/js.tests/test/org/jetbrains/kotlin/js/test/AbstractDceTest.kt

@@ -25,8 +25,8 @@ abstract class AbstractDceTest : TestCase() {
     fun doTest(filePath: String) {
         val file = File(filePath)
         val fileContents = file.readText()
-        val inputFile = InputFile(filePath, File(pathToOutputDir, file.relativeTo(File(pathToTestDir)).path).path, "main")
-        val dceResult = DeadCodeElimination.run(setOf(inputFile), extractDeclarations(REQUEST_REACHABLE_PATTERN, fileContents)) {}
+        val inputFile = InputFile(filePath, null, File(pathToOutputDir, file.relativeTo(File(pathToTestDir)).path).path, "main")
+        val dceResult = DeadCodeElimination.run(setOf(inputFile), extractDeclarations(REQUEST_REACHABLE_PATTERN, fileContents)) { _, _ -> }
         val reachableNodeStrings = dceResult.reachableNodes.map { it.toString().removePrefix("<unknown>.") }.toSet()
 
         for (assertedDeclaration in extractDeclarations(ASSERT_REACHABLE_PATTERN, fileContents)) {

js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt

@@ -440,7 +440,7 @@ abstract class BasicBoxTest(
         val codeWithLines = generatedProgram.toStringWithLineNumbers()
 
         val parsedProgram = JsProgram()
-        parsedProgram.globalBlock.statements += parse(code, ThrowExceptionOnErrorReporter, parsedProgram.scope, outputFile.path)
+        parsedProgram.globalBlock.statements += parse(code, ThrowExceptionOnErrorReporter, parsedProgram.scope, outputFile.path).orEmpty()
         removeLocationFromBlocks(parsedProgram)
         val sourceMapParseResult = SourceMapParser.parse(StringReader(generatedSourceMap))
         val sourceMap = when (sourceMapParseResult) {
@@ -524,12 +524,12 @@ abstract class BasicBoxTest(
         val kotlinJsLibOutput = File(workDir, "kotlin.min.js").path
         val kotlinTestJsLibOutput = File(workDir, "kotlin-test.min.js").path
 
-        val kotlinJsInputFile = InputFile(kotlinJsLib, kotlinJsLibOutput, "kotlin")
-        val kotlinTestJsInputFile = InputFile(kotlinTestJsLib, kotlinTestJsLibOutput, "kotlin-test")
+        val kotlinJsInputFile = InputFile(kotlinJsLib, null, kotlinJsLibOutput, "kotlin")
+        val kotlinTestJsInputFile = InputFile(kotlinTestJsLib, null, kotlinTestJsLibOutput, "kotlin-test")
 
         val filesToMinify = generatedJsFiles.associate { (fileName, module) ->
             val inputFileName = File(fileName).nameWithoutExtension
-            fileName to InputFile(fileName, File(workDir, inputFileName + ".min.js").absolutePath, module.name)
+            fileName to InputFile(fileName, null, File(workDir, inputFileName + ".min.js").absolutePath, module.name)
         }
 
         val testFunctionFqn = testModuleName + (if (testPackage.isNullOrEmpty()) "" else ".$testPackage") + ".$testFunction"
@@ -538,7 +538,7 @@ abstract class BasicBoxTest(
                 "kotlin.kotlin.io.output.buffer", "kotlin-test.kotlin.test.overrideAsserter_wbnzx$"
         )
         val allFilesToMinify = filesToMinify.values + kotlinJsInputFile + kotlinTestJsInputFile
-        val dceResult = DeadCodeElimination.run(allFilesToMinify, additionalReachableNodes) { }
+        val dceResult = DeadCodeElimination.run(allFilesToMinify, additionalReachableNodes) { _, _ -> }
 
         val reachableNodes = dceResult.reachableNodes
         minificationThresholdChecker(reachableNodes.size)

js/js.tests/test/org/jetbrains/kotlin/js/test/ast/NameResolutionTest.kt

@@ -57,8 +57,8 @@ class NameResolutionTest {
         val expectedCode = FileUtil.loadFile(File(expectedName))
 
         val parserScope = JsFunctionScope(JsRootScope(JsProgram()), "<js fun>")
-        val originalAst = JsGlobalBlock().apply { statements += parse(originalCode, errorReporter, parserScope, originalName) }
-        val expectedAst = JsGlobalBlock().apply { statements += parse(expectedCode, errorReporter, parserScope, expectedName) }
+        val originalAst = JsGlobalBlock().apply { statements += parse(originalCode, errorReporter, parserScope, originalName).orEmpty() }
+        val expectedAst = JsGlobalBlock().apply { statements += parse(expectedCode, errorReporter, parserScope, expectedName).orEmpty() }
 
         originalAst.accept(object : RecursiveJsVisitor() {
             val cache = mutableMapOf<JsName, JsName>()

js/js.tests/test/org/jetbrains/kotlin/js/test/optimizer/BasicOptimizerTest.kt

@@ -54,7 +54,7 @@ abstract class BasicOptimizerTest(private var basePath: String) {
 
     private fun checkOptimizer(unoptimizedCode: String, optimizedCode: String) {
         val parserScope = JsFunctionScope(JsRootScope(JsProgram()), "<js fun>")
-        val unoptimizedAst = parse(unoptimizedCode, errorReporter, parserScope, "<unknown file>")
+        val unoptimizedAst = parse(unoptimizedCode, errorReporter, parserScope, "<unknown file>")!!
 
         updateMetadata(unoptimizedCode, unoptimizedAst)
 
@@ -62,7 +62,7 @@ abstract class BasicOptimizerTest(private var basePath: String) {
             process(statement)
         }
 
-        val optimizedAst = parse(optimizedCode, errorReporter, parserScope, "<unknown file>")
+        val optimizedAst = parse(optimizedCode, errorReporter, parserScope, "<unknown file>")!!
         Assert.assertEquals(astToString(optimizedAst), astToString(unoptimizedAst))
     }