JS: add DCE devmode. Fix mapping paths in source maps

See KT-20210, KT-21307

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2JSDceArguments.kt

@@ -40,4 +40,11 @@ class K2JSDceArguments : CommonToolArguments() {
             description = "Print declarations marked as reachable"
     )
     var printReachabilityInfo: Boolean by FreezableVar(false)
+
+    @Argument(
+            value = "-dev-mode",
+            description = "Development mode: don't strip out any code, just copy dependencies"
+    )
+    @GradleOption(DefaultValues.BooleanFalseDefault::class)
+    var devMode: Boolean by FreezableVar(false)
 }

compiler/cli/src/org/jetbrains/kotlin/cli/js/dce/K2JSDce.kt

@@ -23,7 +23,9 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.config.Services
 import org.jetbrains.kotlin.js.dce.*
-import java.io.File
+import org.jetbrains.kotlin.js.inline.util.RelativePathCalculator
+import org.jetbrains.kotlin.js.parser.sourcemaps.*
+import java.io.*
 import java.util.zip.ZipFile
 
 class K2JSDce : CLITool<K2JSDceArguments>() {
@@ -65,6 +67,16 @@ class K2JSDce : CLITool<K2JSDceArguments>() {
             }
         }
 
+        return if (!arguments.devMode) {
+            performDce(files, arguments, messageCollector)
+        }
+        else {
+            copyFiles(files)
+            ExitCode.OK
+        }
+    }
+
+    private fun performDce(files: List<InputFile>, arguments: K2JSDceArguments, messageCollector: MessageCollector): ExitCode {
         val includedDeclarations = arguments.declarationsToKeep.orEmpty().toSet()
 
         val logConsumer = { level: DCELogLevel, message: String ->
@@ -75,6 +87,7 @@ class K2JSDce : CLITool<K2JSDceArguments>() {
             }
             messageCollector.report(severity, message)
         }
+
         val dceResult = DeadCodeElimination.run(files, includedDeclarations, logConsumer)
         if (dceResult.status == DeadCodeEliminationStatus.FAILED) return ExitCode.COMPILATION_ERROR
         val nodes = dceResult.reachableNodes.filterTo(mutableSetOf()) { it.reachable }
@@ -89,6 +102,69 @@ class K2JSDce : CLITool<K2JSDceArguments>() {
         return ExitCode.OK
     }
 
+    private fun copyFiles(files: List<InputFile>) {
+        for (file in files) {
+            copyResource(file.resource, File(file.outputPath))
+            file.sourceMapResource?.let { sourceMap ->
+                val sourceMapTarget = File(file.outputPath + ".map")
+                val inputFile = File(sourceMap.name)
+                if (!inputFile.exists() || !mapSourcePaths(inputFile, sourceMapTarget)) {
+                    copyResource(sourceMap, sourceMapTarget)
+                }
+            }
+        }
+    }
+
+    private fun copyResource(resource: InputResource, targetFile: File) {
+        if (targetFile.exists() && resource.lastModified() < targetFile.lastModified()) return
+
+        targetFile.parentFile.mkdirs()
+        resource.reader().use { input ->
+            FileOutputStream(targetFile).use { output ->
+                input.copyTo(output)
+            }
+         }
+    }
+
+    private fun mapSourcePaths(inputFile: File, targetFile: File): Boolean {
+        val json = try {
+            InputStreamReader(FileInputStream(inputFile), "UTF-8").use { parseJson(it) }
+        }
+        catch (e: JsonSyntaxException) {
+            return false
+        }
+
+        val sourcesArray = (json as? JsonObject)?.properties?.get("sources") as? JsonArray ?: return false
+        val sources = sourcesArray.elements.map {
+            (it as? JsonString)?.value ?: return false
+        }
+
+        val pathCalculator = RelativePathCalculator(targetFile.parentFile)
+        val mappedSources = sources.map {
+            val result = pathCalculator.calculateRelativePathTo(File(inputFile.parentFile, it))
+            if (result != null) {
+                if (File(targetFile.parentFile, result).exists()) {
+                    result
+                }
+                else {
+                    it
+                }
+            }
+            else {
+                it
+            }
+        }
+
+        if (mappedSources == sources) return false
+
+        json.properties["sources"] = JsonArray(*mappedSources.map { JsonString(it) }.toTypedArray())
+
+        targetFile.parentFile.mkdirs()
+        OutputStreamWriter(FileOutputStream(targetFile), "UTF-8").use { it.write(json.toString()) }
+
+        return true
+    }
+
     private fun collectInputFiles(baseDir: File, fileName: String, messageCollector: MessageCollector): List<InputFile>? {
         val file = File(fileName)
         return when {

compiler/testData/cli/js-dce/dceHelp.out

@@ -2,6 +2,7 @@ Usage: kotlin-dce-js <options> <source files>
 where possible options include:
   -keep <fully.qualified.name[,]>
                              List of fully-qualified names of declarations that shouldn't be eliminated
+  -dev-mode                  Development mode: don't strip out any code, just copy dependencies
   -output-dir <path>         Output directory
   -Werror                    Report an error if there are any warnings
   -X                         Print a synopsis of advanced options

js/js.dce/src/org/jetbrains/kotlin/js/dce/DeadCodeElimination.kt

@@ -120,8 +120,12 @@ class DeadCodeElimination(private val logConsumer: (DCELogLevel, String) -> Unit
                 val sourceMapFile = File(file.outputPath + ".map")
                 val textOutput = TextOutputImpl()
                 val sourceMapBuilder = SourceMap3Builder(File(file.outputPath), textOutput, "")
-                val sourcePathResolver = SourceFilePathResolver(mutableListOf(), File(file.outputPath).parentFile)
+
-                val consumer = SourceMapBuilderConsumer(sourceMapBuilder, sourcePathResolver, true, true)
+                val inputFile = File(file.resource.name)
+                val sourceBaseDir = if (inputFile.exists()) inputFile.parentFile else File(".")
+
+                val sourcePathResolver = SourceFilePathResolver(emptyList(), File(file.outputPath).parentFile)
+                val consumer = SourceMapBuilderConsumer(sourceBaseDir, sourceMapBuilder, sourcePathResolver, true, true)
                 block.accept(JsToStringGenerationVisitor(textOutput, consumer))
                 val sourceMapContent = sourceMapBuilder.build()
                 sourceMapBuilder.addLink()

js/js.dce/src/org/jetbrains/kotlin/js/dce/InputResource.kt

@@ -21,13 +21,19 @@ import java.io.FileInputStream
 import java.io.InputStream
 import java.util.zip.ZipFile
 
-class InputResource(val name: String, val reader: () -> InputStream) {
+class InputResource(val name: String, val lastModified: () -> Long, val reader: () -> InputStream) {
     companion object {
-        fun file(path: String): InputResource = InputResource(path) { FileInputStream(File(path)) }
+        fun file(path: String): InputResource = InputResource(path, { File(path).lastModified() }) { FileInputStream(File(path)) }
 
-        fun zipFile(path: String, entryPath: String): InputResource = InputResource("$path!$entryPath") {
+        fun zipFile(path: String, entryPath: String): InputResource =
-            val zipFile = ZipFile(path)
+                InputResource("$path!$entryPath", { getZipModificationTime(path, entryPath) }) {
-            zipFile.getInputStream(zipFile.getEntry(entryPath))
+                    val zipFile = ZipFile(path)
+                    zipFile.getInputStream(zipFile.getEntry(entryPath))
+                }
+
+        private fun getZipModificationTime(path: String, entryPath: String): Long {
+            val result = ZipFile(path).getEntry(entryPath).time
+            return if (result != -1L) result else File(path).lastModified()
         }
     }
 }

js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt

@@ -460,7 +460,8 @@ abstract class BasicBoxTest(
         val output = TextOutputImpl()
         val pathResolver = SourceFilePathResolver(mutableListOf(File(".")), null)
         val sourceMapBuilder = SourceMap3Builder(outputFile, output, "")
-        generatedProgram.accept(JsToStringGenerationVisitor(output, SourceMapBuilderConsumer(sourceMapBuilder, pathResolver, false, false)))
+        generatedProgram.accept(JsToStringGenerationVisitor(
+                output, SourceMapBuilderConsumer(File("."), sourceMapBuilder, pathResolver, false, false)))
         val code = output.toString()
         val generatedSourceMap = sourceMapBuilder.build()
 

js/js.translator/src/org/jetbrains/kotlin/js/facade/SourceMapBuilderConsumer.java

@@ -34,6 +34,9 @@ import java.util.List;
 import java.util.function.Supplier;
 
 public class SourceMapBuilderConsumer implements SourceLocationConsumer {
+    @NotNull
+    private final File sourceBaseDir;
+
     @NotNull
     private final SourceMapMappingConsumer mappingConsumer;
 
@@ -48,10 +51,12 @@ public class SourceMapBuilderConsumer implements SourceLocationConsumer {
     private final List<Object> sourceStack = new ArrayList<>();
 
     public SourceMapBuilderConsumer(
+            @NotNull File sourceBaseDir,
             @NotNull SourceMapMappingConsumer mappingConsumer,
             @NotNull SourceFilePathResolver pathResolver,
             boolean provideCurrentModuleContent, boolean provideExternalModuleContent
     ) {
+        this.sourceBaseDir = sourceBaseDir;
         this.mappingConsumer = mappingConsumer;
         this.pathResolver = pathResolver;
         this.provideCurrentModuleContent = provideCurrentModuleContent;
@@ -109,7 +114,23 @@ public class SourceMapBuilderConsumer implements SourceLocationConsumer {
         else if (sourceInfo instanceof JsLocationWithSource) {
             JsLocationWithSource location = (JsLocationWithSource) sourceInfo;
             Supplier<Reader> contentSupplier = provideExternalModuleContent ? location.getSourceProvider()::invoke : () -> null;
-            mappingConsumer.addMapping(location.getFile(), location.getIdentityObject(), contentSupplier,
+            String path;
+
+            File absFile = new File(location.getFile()).isAbsolute() ?
+                           new File(location.getFile()) :
+                           new File(sourceBaseDir, location.getFile());
+            if (absFile.isAbsolute()) {
+                try {
+                    path = pathResolver.getPathRelativeToSourceRoots(absFile);
+                }
+                catch (IOException e) {
+                    path = location.getFile();
+                }
+            }
+            else {
+                path = location.getFile();
+            }
+            mappingConsumer.addMapping(path, location.getIdentityObject(), contentSupplier,
                                location.getStartLine(), location.getStartChar());
         }
     }

js/js.translator/src/org/jetbrains/kotlin/js/facade/TranslationResult.kt

@@ -69,6 +69,7 @@ abstract class TranslationResult protected constructor(val diagnostics: Diagnost
                         val sourceMapContentEmbedding = config.sourceMapContentEmbedding
                         val pathResolver = SourceFilePathResolver.create(config)
                         SourceMapBuilderConsumer(
+                                File("."),
                                 sourceMapBuilder,
                                 pathResolver,
                                 sourceMapContentEmbedding == SourceMapSourceEmbedding.ALWAYS,

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/Kotlin2JsGradlePluginIT.kt

@@ -2,7 +2,6 @@ package org.jetbrains.kotlin.gradle
 
 import org.gradle.api.logging.LogLevel
 import org.jetbrains.kotlin.gradle.tasks.USING_EXPERIMENTAL_JS_INCREMENTAL_COMPILATION_MESSAGE
-import org.jetbrains.kotlin.gradle.util.allKotlinFiles
 import org.jetbrains.kotlin.gradle.util.getFileByName
 import org.jetbrains.kotlin.gradle.util.modify
 import org.junit.Test
@@ -223,11 +222,36 @@ class Kotlin2JsGradlePluginIT : BaseGradleIT() {
         val project = Project("kotlin2JsDceProject", "2.10", minLogLevel = LogLevel.INFO)
 
         project.build("runRhino") {
-            println(output)
             assertSuccessful()
             val pathPrefix = "mainProject/build/min"
             assertFileExists("$pathPrefix/exampleapp.js.map")
             assertFileExists("$pathPrefix/examplelib.js.map")
+            assertFileContains("$pathPrefix/exampleapp.js.map", "\"../../src/main/kotlin/exampleapp/main.kt\"")
+
+            assertFileExists("$pathPrefix/kotlin.js")
+            assertTrue(fileInWorkingDir("$pathPrefix/kotlin.js").length() < 500 * 1000, "Looks like kotlin.js file was not minified by DCE")
+        }
+    }
+
+    @Test
+    fun testDceDevMode() {
+        val project = Project("kotlin2JsDceProject", "2.10", minLogLevel = LogLevel.INFO)
+
+        project.setupWorkingDir()
+        File(project.projectDir, "mainProject/build.gradle").modify {
+            it + "\n" +
+            "runDceKotlinJs.dceOptions.devMode = true\n"
+        }
+
+        project.build("runRhino") {
+            assertSuccessful()
+            val pathPrefix = "mainProject/build/min"
+            assertFileExists("$pathPrefix/exampleapp.js.map")
+            assertFileExists("$pathPrefix/examplelib.js.map")
+            assertFileContains("$pathPrefix/exampleapp.js.map", "\"../../src/main/kotlin/exampleapp/main.kt\"")
+
+            assertFileExists("$pathPrefix/kotlin.js")
+            assertTrue(fileInWorkingDir("$pathPrefix/kotlin.js").length() > 1000 * 1000, "Looks like kotlin.js file was minified by DCE")
         }
     }
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinJsDceOptions.kt

@@ -3,4 +3,10 @@
 package org.jetbrains.kotlin.gradle.dsl
 
 interface KotlinJsDceOptions  : org.jetbrains.kotlin.gradle.dsl.KotlinCommonToolOptions {
+
+    /**
+     * Development mode: don't strip out any code, just copy dependencies and remap source maps
+     * Default value: false
+     */
+     var devMode: kotlin.Boolean
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinJsDceOptionsBase.kt

@@ -19,10 +19,16 @@ internal abstract class KotlinJsDceOptionsBase : org.jetbrains.kotlin.gradle.dsl
         get() = verboseField ?: false
         set(value) { verboseField = value }
 
+    private var devModeField: kotlin.Boolean? = null
+    override var devMode: kotlin.Boolean
+        get() = devModeField ?: false
+        set(value) { devModeField = value }
+
     internal open fun updateArguments(args: org.jetbrains.kotlin.cli.common.arguments.K2JSDceArguments) {
         allWarningsAsErrorsField?.let { args.allWarningsAsErrors = it }
         suppressWarningsField?.let { args.suppressWarnings = it }
         verboseField?.let { args.verbose = it }
+        devModeField?.let { args.devMode = it }
     }
 }
 
@@ -30,4 +36,5 @@ internal fun org.jetbrains.kotlin.cli.common.arguments.K2JSDceArguments.fillDefa
     allWarningsAsErrors = false
     suppressWarnings = false
     verbose = false
+    devMode = false
 }