From 5672e1f56ecec05629c56b911d8a93bebf640649 Mon Sep 17 00:00:00 2001 From: Alexey Andreev Date: Tue, 21 Nov 2017 19:29:26 +0300 Subject: [PATCH] JS: add DCE devmode. Fix mapping paths in source maps See KT-20210, KT-21307 --- .../cli/common/arguments/K2JSDceArguments.kt | 7 ++ .../jetbrains/kotlin/cli/js/dce/K2JSDce.kt | 78 ++++++++++++++++++- compiler/testData/cli/js-dce/dceHelp.out | 1 + .../kotlin/js/dce/DeadCodeElimination.kt | 8 +- .../jetbrains/kotlin/js/dce/InputResource.kt | 16 ++-- .../jetbrains/kotlin/js/test/BasicBoxTest.kt | 3 +- .../js/facade/SourceMapBuilderConsumer.java | 23 +++++- .../kotlin/js/facade/TranslationResult.kt | 1 + .../kotlin/gradle/Kotlin2JsGradlePluginIT.kt | 28 ++++++- .../kotlin/gradle/dsl/KotlinJsDceOptions.kt | 6 ++ .../gradle/dsl/KotlinJsDceOptionsBase.kt | 7 ++ 11 files changed, 166 insertions(+), 12 deletions(-) diff --git a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2JSDceArguments.kt b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2JSDceArguments.kt index 53ca5a52444..6996315a43c 100644 --- a/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2JSDceArguments.kt +++ b/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2JSDceArguments.kt @@ -40,4 +40,11 @@ class K2JSDceArguments : CommonToolArguments() { description = "Print declarations marked as reachable" ) var printReachabilityInfo: Boolean by FreezableVar(false) + + @Argument( + value = "-dev-mode", + description = "Development mode: don't strip out any code, just copy dependencies" + ) + @GradleOption(DefaultValues.BooleanFalseDefault::class) + var devMode: Boolean by FreezableVar(false) } diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/js/dce/K2JSDce.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/js/dce/K2JSDce.kt index 28f6cdcf9ee..420017f4259 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/js/dce/K2JSDce.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/js/dce/K2JSDce.kt @@ -23,7 +23,9 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity import org.jetbrains.kotlin.cli.common.messages.MessageCollector import org.jetbrains.kotlin.config.Services import org.jetbrains.kotlin.js.dce.* -import java.io.File +import org.jetbrains.kotlin.js.inline.util.RelativePathCalculator +import org.jetbrains.kotlin.js.parser.sourcemaps.* +import java.io.* import java.util.zip.ZipFile class K2JSDce : CLITool() { @@ -65,6 +67,16 @@ class K2JSDce : CLITool() { } } + return if (!arguments.devMode) { + performDce(files, arguments, messageCollector) + } + else { + copyFiles(files) + ExitCode.OK + } + } + + private fun performDce(files: List, arguments: K2JSDceArguments, messageCollector: MessageCollector): ExitCode { val includedDeclarations = arguments.declarationsToKeep.orEmpty().toSet() val logConsumer = { level: DCELogLevel, message: String -> @@ -75,6 +87,7 @@ class K2JSDce : CLITool() { } messageCollector.report(severity, message) } + val dceResult = DeadCodeElimination.run(files, includedDeclarations, logConsumer) if (dceResult.status == DeadCodeEliminationStatus.FAILED) return ExitCode.COMPILATION_ERROR val nodes = dceResult.reachableNodes.filterTo(mutableSetOf()) { it.reachable } @@ -89,6 +102,69 @@ class K2JSDce : CLITool() { return ExitCode.OK } + private fun copyFiles(files: List) { + for (file in files) { + copyResource(file.resource, File(file.outputPath)) + file.sourceMapResource?.let { sourceMap -> + val sourceMapTarget = File(file.outputPath + ".map") + val inputFile = File(sourceMap.name) + if (!inputFile.exists() || !mapSourcePaths(inputFile, sourceMapTarget)) { + copyResource(sourceMap, sourceMapTarget) + } + } + } + } + + private fun copyResource(resource: InputResource, targetFile: File) { + if (targetFile.exists() && resource.lastModified() < targetFile.lastModified()) return + + targetFile.parentFile.mkdirs() + resource.reader().use { input -> + FileOutputStream(targetFile).use { output -> + input.copyTo(output) + } + } + } + + private fun mapSourcePaths(inputFile: File, targetFile: File): Boolean { + val json = try { + InputStreamReader(FileInputStream(inputFile), "UTF-8").use { parseJson(it) } + } + catch (e: JsonSyntaxException) { + return false + } + + val sourcesArray = (json as? JsonObject)?.properties?.get("sources") as? JsonArray ?: return false + val sources = sourcesArray.elements.map { + (it as? JsonString)?.value ?: return false + } + + val pathCalculator = RelativePathCalculator(targetFile.parentFile) + val mappedSources = sources.map { + val result = pathCalculator.calculateRelativePathTo(File(inputFile.parentFile, it)) + if (result != null) { + if (File(targetFile.parentFile, result).exists()) { + result + } + else { + it + } + } + else { + it + } + } + + if (mappedSources == sources) return false + + json.properties["sources"] = JsonArray(*mappedSources.map { JsonString(it) }.toTypedArray()) + + targetFile.parentFile.mkdirs() + OutputStreamWriter(FileOutputStream(targetFile), "UTF-8").use { it.write(json.toString()) } + + return true + } + private fun collectInputFiles(baseDir: File, fileName: String, messageCollector: MessageCollector): List? { val file = File(fileName) return when { diff --git a/compiler/testData/cli/js-dce/dceHelp.out b/compiler/testData/cli/js-dce/dceHelp.out index ce88b473542..c4bec792ee1 100644 --- a/compiler/testData/cli/js-dce/dceHelp.out +++ b/compiler/testData/cli/js-dce/dceHelp.out @@ -2,6 +2,7 @@ Usage: kotlin-dce-js where possible options include: -keep List of fully-qualified names of declarations that shouldn't be eliminated + -dev-mode Development mode: don't strip out any code, just copy dependencies -output-dir Output directory -Werror Report an error if there are any warnings -X Print a synopsis of advanced options diff --git a/js/js.dce/src/org/jetbrains/kotlin/js/dce/DeadCodeElimination.kt b/js/js.dce/src/org/jetbrains/kotlin/js/dce/DeadCodeElimination.kt index 7fbe068073b..136c629f011 100644 --- a/js/js.dce/src/org/jetbrains/kotlin/js/dce/DeadCodeElimination.kt +++ b/js/js.dce/src/org/jetbrains/kotlin/js/dce/DeadCodeElimination.kt @@ -120,8 +120,12 @@ class DeadCodeElimination(private val logConsumer: (DCELogLevel, String) -> Unit val sourceMapFile = File(file.outputPath + ".map") val textOutput = TextOutputImpl() val sourceMapBuilder = SourceMap3Builder(File(file.outputPath), textOutput, "") - val sourcePathResolver = SourceFilePathResolver(mutableListOf(), File(file.outputPath).parentFile) - val consumer = SourceMapBuilderConsumer(sourceMapBuilder, sourcePathResolver, true, true) + + val inputFile = File(file.resource.name) + val sourceBaseDir = if (inputFile.exists()) inputFile.parentFile else File(".") + + val sourcePathResolver = SourceFilePathResolver(emptyList(), File(file.outputPath).parentFile) + val consumer = SourceMapBuilderConsumer(sourceBaseDir, sourceMapBuilder, sourcePathResolver, true, true) block.accept(JsToStringGenerationVisitor(textOutput, consumer)) val sourceMapContent = sourceMapBuilder.build() sourceMapBuilder.addLink() diff --git a/js/js.dce/src/org/jetbrains/kotlin/js/dce/InputResource.kt b/js/js.dce/src/org/jetbrains/kotlin/js/dce/InputResource.kt index f84d84c7190..9ddd33670a4 100644 --- a/js/js.dce/src/org/jetbrains/kotlin/js/dce/InputResource.kt +++ b/js/js.dce/src/org/jetbrains/kotlin/js/dce/InputResource.kt @@ -21,13 +21,19 @@ import java.io.FileInputStream import java.io.InputStream import java.util.zip.ZipFile -class InputResource(val name: String, val reader: () -> InputStream) { +class InputResource(val name: String, val lastModified: () -> Long, val reader: () -> InputStream) { companion object { - fun file(path: String): InputResource = InputResource(path) { FileInputStream(File(path)) } + fun file(path: String): InputResource = InputResource(path, { File(path).lastModified() }) { FileInputStream(File(path)) } - fun zipFile(path: String, entryPath: String): InputResource = InputResource("$path!$entryPath") { - val zipFile = ZipFile(path) - zipFile.getInputStream(zipFile.getEntry(entryPath)) + fun zipFile(path: String, entryPath: String): InputResource = + InputResource("$path!$entryPath", { getZipModificationTime(path, entryPath) }) { + val zipFile = ZipFile(path) + zipFile.getInputStream(zipFile.getEntry(entryPath)) + } + + private fun getZipModificationTime(path: String, entryPath: String): Long { + val result = ZipFile(path).getEntry(entryPath).time + return if (result != -1L) result else File(path).lastModified() } } } \ No newline at end of file diff --git a/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt b/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt index 909410373c5..dc6826b9817 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt @@ -460,7 +460,8 @@ abstract class BasicBoxTest( val output = TextOutputImpl() val pathResolver = SourceFilePathResolver(mutableListOf(File(".")), null) val sourceMapBuilder = SourceMap3Builder(outputFile, output, "") - generatedProgram.accept(JsToStringGenerationVisitor(output, SourceMapBuilderConsumer(sourceMapBuilder, pathResolver, false, false))) + generatedProgram.accept(JsToStringGenerationVisitor( + output, SourceMapBuilderConsumer(File("."), sourceMapBuilder, pathResolver, false, false))) val code = output.toString() val generatedSourceMap = sourceMapBuilder.build() diff --git a/js/js.translator/src/org/jetbrains/kotlin/js/facade/SourceMapBuilderConsumer.java b/js/js.translator/src/org/jetbrains/kotlin/js/facade/SourceMapBuilderConsumer.java index 74aaee2553c..305dbb33fbe 100644 --- a/js/js.translator/src/org/jetbrains/kotlin/js/facade/SourceMapBuilderConsumer.java +++ b/js/js.translator/src/org/jetbrains/kotlin/js/facade/SourceMapBuilderConsumer.java @@ -34,6 +34,9 @@ import java.util.List; import java.util.function.Supplier; public class SourceMapBuilderConsumer implements SourceLocationConsumer { + @NotNull + private final File sourceBaseDir; + @NotNull private final SourceMapMappingConsumer mappingConsumer; @@ -48,10 +51,12 @@ public class SourceMapBuilderConsumer implements SourceLocationConsumer { private final List sourceStack = new ArrayList<>(); public SourceMapBuilderConsumer( + @NotNull File sourceBaseDir, @NotNull SourceMapMappingConsumer mappingConsumer, @NotNull SourceFilePathResolver pathResolver, boolean provideCurrentModuleContent, boolean provideExternalModuleContent ) { + this.sourceBaseDir = sourceBaseDir; this.mappingConsumer = mappingConsumer; this.pathResolver = pathResolver; this.provideCurrentModuleContent = provideCurrentModuleContent; @@ -109,7 +114,23 @@ public class SourceMapBuilderConsumer implements SourceLocationConsumer { else if (sourceInfo instanceof JsLocationWithSource) { JsLocationWithSource location = (JsLocationWithSource) sourceInfo; Supplier contentSupplier = provideExternalModuleContent ? location.getSourceProvider()::invoke : () -> null; - mappingConsumer.addMapping(location.getFile(), location.getIdentityObject(), contentSupplier, + String path; + + File absFile = new File(location.getFile()).isAbsolute() ? + new File(location.getFile()) : + new File(sourceBaseDir, location.getFile()); + if (absFile.isAbsolute()) { + try { + path = pathResolver.getPathRelativeToSourceRoots(absFile); + } + catch (IOException e) { + path = location.getFile(); + } + } + else { + path = location.getFile(); + } + mappingConsumer.addMapping(path, location.getIdentityObject(), contentSupplier, location.getStartLine(), location.getStartChar()); } } diff --git a/js/js.translator/src/org/jetbrains/kotlin/js/facade/TranslationResult.kt b/js/js.translator/src/org/jetbrains/kotlin/js/facade/TranslationResult.kt index 9f71796a876..b97f2dfde91 100644 --- a/js/js.translator/src/org/jetbrains/kotlin/js/facade/TranslationResult.kt +++ b/js/js.translator/src/org/jetbrains/kotlin/js/facade/TranslationResult.kt @@ -69,6 +69,7 @@ abstract class TranslationResult protected constructor(val diagnostics: Diagnost val sourceMapContentEmbedding = config.sourceMapContentEmbedding val pathResolver = SourceFilePathResolver.create(config) SourceMapBuilderConsumer( + File("."), sourceMapBuilder, pathResolver, sourceMapContentEmbedding == SourceMapSourceEmbedding.ALWAYS, diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/Kotlin2JsGradlePluginIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/Kotlin2JsGradlePluginIT.kt index fc9ca060314..c8767045ffa 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/Kotlin2JsGradlePluginIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/Kotlin2JsGradlePluginIT.kt @@ -2,7 +2,6 @@ package org.jetbrains.kotlin.gradle import org.gradle.api.logging.LogLevel import org.jetbrains.kotlin.gradle.tasks.USING_EXPERIMENTAL_JS_INCREMENTAL_COMPILATION_MESSAGE -import org.jetbrains.kotlin.gradle.util.allKotlinFiles import org.jetbrains.kotlin.gradle.util.getFileByName import org.jetbrains.kotlin.gradle.util.modify import org.junit.Test @@ -223,11 +222,36 @@ class Kotlin2JsGradlePluginIT : BaseGradleIT() { val project = Project("kotlin2JsDceProject", "2.10", minLogLevel = LogLevel.INFO) project.build("runRhino") { - println(output) assertSuccessful() val pathPrefix = "mainProject/build/min" assertFileExists("$pathPrefix/exampleapp.js.map") assertFileExists("$pathPrefix/examplelib.js.map") + assertFileContains("$pathPrefix/exampleapp.js.map", "\"../../src/main/kotlin/exampleapp/main.kt\"") + + assertFileExists("$pathPrefix/kotlin.js") + assertTrue(fileInWorkingDir("$pathPrefix/kotlin.js").length() < 500 * 1000, "Looks like kotlin.js file was not minified by DCE") + } + } + + @Test + fun testDceDevMode() { + val project = Project("kotlin2JsDceProject", "2.10", minLogLevel = LogLevel.INFO) + + project.setupWorkingDir() + File(project.projectDir, "mainProject/build.gradle").modify { + it + "\n" + + "runDceKotlinJs.dceOptions.devMode = true\n" + } + + project.build("runRhino") { + assertSuccessful() + val pathPrefix = "mainProject/build/min" + assertFileExists("$pathPrefix/exampleapp.js.map") + assertFileExists("$pathPrefix/examplelib.js.map") + assertFileContains("$pathPrefix/exampleapp.js.map", "\"../../src/main/kotlin/exampleapp/main.kt\"") + + assertFileExists("$pathPrefix/kotlin.js") + assertTrue(fileInWorkingDir("$pathPrefix/kotlin.js").length() > 1000 * 1000, "Looks like kotlin.js file was minified by DCE") } } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinJsDceOptions.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinJsDceOptions.kt index 3ed746e94c7..c0db6250847 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinJsDceOptions.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinJsDceOptions.kt @@ -3,4 +3,10 @@ package org.jetbrains.kotlin.gradle.dsl interface KotlinJsDceOptions : org.jetbrains.kotlin.gradle.dsl.KotlinCommonToolOptions { + + /** + * Development mode: don't strip out any code, just copy dependencies and remap source maps + * Default value: false + */ + var devMode: kotlin.Boolean } diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinJsDceOptionsBase.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinJsDceOptionsBase.kt index 279b1f4868d..92d0ae54837 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinJsDceOptionsBase.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinJsDceOptionsBase.kt @@ -19,10 +19,16 @@ internal abstract class KotlinJsDceOptionsBase : org.jetbrains.kotlin.gradle.dsl get() = verboseField ?: false set(value) { verboseField = value } + private var devModeField: kotlin.Boolean? = null + override var devMode: kotlin.Boolean + get() = devModeField ?: false + set(value) { devModeField = value } + internal open fun updateArguments(args: org.jetbrains.kotlin.cli.common.arguments.K2JSDceArguments) { allWarningsAsErrorsField?.let { args.allWarningsAsErrors = it } suppressWarningsField?.let { args.suppressWarnings = it } verboseField?.let { args.verbose = it } + devModeField?.let { args.devMode = it } } } @@ -30,4 +36,5 @@ internal fun org.jetbrains.kotlin.cli.common.arguments.K2JSDceArguments.fillDefa allWarningsAsErrors = false suppressWarnings = false verbose = false + devMode = false }