JS: change the way how DCE resolves input paths

compiler/cli/src/org/jetbrains/kotlin/cli/js/dce/K2JSDce.kt

@@ -24,28 +24,45 @@ import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.config.Services
 import org.jetbrains.kotlin.js.dce.*
 import java.io.File
+import java.util.zip.ZipFile
 
 class K2JSDce : CLITool<K2JSDceArguments>() {
     override fun createArguments(): K2JSDceArguments = K2JSDceArguments()
 
     override fun execImpl(messageCollector: MessageCollector, services: Services, arguments: K2JSDceArguments): ExitCode {
         val baseDir = File(arguments.outputDirectory ?: "min")
-        val files = arguments.freeArgs.map { arg ->
+        var hasErrors = false
-            val parts = arg.split(File.pathSeparator, ignoreCase = false, limit = 2)
+        val files = arguments.freeArgs.flatMap { arg ->
-            val inputName = parts[0]
+            val files = collectInputFiles(baseDir, arg, messageCollector)
-            val moduleName = parts.getOrNull(1) ?: ""
+            if (files != null) {
-            val resolvedModuleName = if (!moduleName.isEmpty()) moduleName else File(inputName).nameWithoutExtension
+                files
-            val pathToSourceMapCandidate = "$inputName.map"
+            }
-            val pathToSourceMap = if (File(pathToSourceMapCandidate).exists()) pathToSourceMapCandidate else null
+            else {
-            InputFile(inputName, pathToSourceMap, File(baseDir, resolvedModuleName + ".js").absolutePath, resolvedModuleName)
+                hasErrors = true
+                emptyList()
+            }
         }
 
+        if (hasErrors) return ExitCode.COMPILATION_ERROR
+
         if (files.isEmpty() && !arguments.version) {
             messageCollector.report(CompilerMessageSeverity.ERROR, "no source files")
             return ExitCode.COMPILATION_ERROR
         }
-        if (!checkSourceFiles(messageCollector, files)) {
+
-            return ExitCode.COMPILATION_ERROR
+        val existingFiles = mutableMapOf<String, InputFile>()
+        for (file in files) {
+            existingFiles[file.outputPath]?.let {
+                messageCollector.report(
+                        CompilerMessageSeverity.ERROR,
+                        "duplicate target file will be created for '${file.resource.name}' and '${it.resource.name}'")
+                return ExitCode.COMPILATION_ERROR
+            }
+            existingFiles[file.outputPath] = file
+            if (File(file.outputPath).isDirectory) {
+                messageCollector.report(CompilerMessageSeverity.ERROR, "cannot open output file '${file.outputPath}': it is a directory")
+                return ExitCode.COMPILATION_ERROR
+            }
         }
 
         val includedDeclarations = arguments.declarationsToKeep.orEmpty().toSet()
@@ -72,35 +89,83 @@ class K2JSDce : CLITool<K2JSDceArguments>() {
         return ExitCode.OK
     }
 
-    private fun checkSourceFiles(messageCollector: MessageCollector, files: List<InputFile>): Boolean {
+    private fun collectInputFiles(baseDir: File, fileName: String, messageCollector: MessageCollector): List<InputFile>? {
-        return files.fold(true) { ok, file ->
+        val file = File(fileName)
-            val inputFile = File(file.path)
+        return when {
-            val outputFile = File(file.outputPath)
+            file.isDirectory -> {
-
+                collectInputFilesFromDirectory(baseDir, fileName)
-            val inputOk = when {
-                !inputFile.exists() -> {
-                    messageCollector.report(CompilerMessageSeverity.ERROR, "source file or directory not found: " + file.path)
-                    false
-                }
-                inputFile.isDirectory -> {
-                    messageCollector.report(CompilerMessageSeverity.ERROR, "input file '" + file.path + "' is a directory")
-                    false
-                }
-                else -> true
             }
-
+            file.isFile -> {
-            val outputOk = when {
+                when {
-                outputFile.exists() && outputFile.isDirectory -> {
+                    fileName.endsWith(".js") -> {
-                    messageCollector.report(CompilerMessageSeverity.ERROR, "cannot open output file '${outputFile.path}': is a directory")
+                        listOf(singleInputFile(baseDir, fileName))
-                    false
+                    }
+                    fileName.endsWith(".zip") || fileName.endsWith(".jar") -> {
+                        collectInputFilesFromZip(baseDir, fileName)
+                    }
+                    else -> {
+                        messageCollector.report(CompilerMessageSeverity.ERROR,
+                                                "invalid file name '$fileName'; must end either with '.js', '.zip' or '.jar'")
+                        null
+                    }
                 }
-                else -> true
             }
-
+            else -> {
-            ok and inputOk and outputOk
+                messageCollector.report(CompilerMessageSeverity.ERROR, "source file or directory not found: $fileName")
+                null
+            }
         }
     }
 
+    private fun singleInputFile(baseDir: File, path: String): InputFile {
+        val moduleName = getModuleNameFromPath(path)
+        val pathToSourceMapCandidate = "$path.map"
+        val pathToSourceMap = if (File(pathToSourceMapCandidate).exists()) pathToSourceMapCandidate else null
+        return InputFile(InputResource.file(path), pathToSourceMap?.let { InputResource.file(it) },
+                         File(baseDir, "$moduleName.js").absolutePath, moduleName)
+    }
+
+    private fun collectInputFilesFromZip(baseDir: File, path: String): List<InputFile> {
+        return ZipFile(path).use { zipFile ->
+            zipFile.entries().asSequence()
+                    .filter { !it.isDirectory }
+                    .filter { it.name.endsWith(".js") }
+                    .filter { zipFile.getEntry(it.name.metaJs()) != null }
+                    .distinctBy { it.name }
+                    .map { entry ->
+                        val moduleName = getModuleNameFromPath(entry.name)
+                        val pathToSourceMapCandidate = "${entry.name}.map"
+                        val pathToSourceMap = if (zipFile.getEntry(pathToSourceMapCandidate) != null) pathToSourceMapCandidate else null
+                        InputFile(InputResource.zipFile(path, entry.name), pathToSourceMap?.let { InputResource.zipFile(path, it) },
+                                  File(baseDir, "$moduleName.js").absolutePath, moduleName)
+                    }
+                    .toList()
+        }
+    }
+
+    private fun collectInputFilesFromDirectory(baseDir: File, path: String): List<InputFile> {
+        return File(path).walkTopDown().asSequence()
+                .filter { !it.isDirectory }
+                .filter { it.name.endsWith(".js") }
+                .filter { File(it.path.metaJs()).exists() }
+                .map { entry ->
+                    val moduleName = getModuleNameFromPath(entry.name)
+                    val pathToSourceMapCandidate = "${entry.path}.map"
+                    val pathToSourceMap = if (File(pathToSourceMapCandidate).exists()) pathToSourceMapCandidate else null
+                    InputFile(InputResource.file(entry.path), pathToSourceMap?.let { InputResource.file(it) },
+                              File(baseDir, "$moduleName.js").absolutePath, moduleName)
+                }
+                .toList()
+    }
+
+    private fun String.metaJs() = removeSuffix(".js") + ".meta.js"
+
+    private fun getModuleNameFromPath(path: String): String {
+        val dotIndex = path.lastIndexOf('.')
+        val slashIndex = maxOf(path.lastIndexOf('/'), path.lastIndexOf('\\'))
+        return path.substring(slashIndex + 1, if (dotIndex < 0) path.length else dotIndex)
+    }
+
     override fun executableScriptFileName(): String = "kotlin-dce-js"
 
     companion object {

compiler/testData/cli/js-dce/notFile.out

@@ -1,2 +1,2 @@
-error: input file 'compiler/testData/cli/js-dce' is a directory
+error: no source files
 COMPILATION_ERROR

compiler/testData/cli/js-dce/outputIsDirectory.out

@@ -1,2 +1,2 @@
-error: cannot open output file '$TESTDATA_DIR$/min/simple.js': is a directory
+error: cannot open output file '$TESTDATA_DIR$/min/simple.js': it is a directory
 COMPILATION_ERROR

compiler/testData/cli/js-dce/overrideOutputName.args

@@ -1,3 +0,0 @@
-$TESTDATA_DIR$/simple.js:bar
--output-dir
-$TEMP_DIR$/min

compiler/testData/cli/js-dce/overrideOutputName.out

@@ -1 +0,0 @@
-OK

compiler/testData/cli/js-dce/overrideOutputName.test

@@ -1 +0,0 @@
-// EXISTS: min/bar.js

compiler/tests/org/jetbrains/kotlin/cli/CliTestGenerated.java

@@ -800,12 +800,6 @@ public class CliTestGenerated extends AbstractCliTest {
             doJsDceTest(fileName);
         }
 
-        @TestMetadata("overrideOutputName.args")
-        public void testOverrideOutputName() throws Exception {
-            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/cli/js-dce/overrideOutputName.args");
-            doJsDceTest(fileName);
-        }
-
         @TestMetadata("parseError.args")
         public void testParseError() throws Exception {
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/cli/js-dce/parseError.args");

js/js.dce/src/org/jetbrains/kotlin/js/dce/DeadCodeElimination.kt

@@ -19,7 +19,10 @@ package org.jetbrains.kotlin.js.dce
 import com.google.gwt.dev.js.rhino.CodePosition
 import com.google.gwt.dev.js.rhino.ErrorReporter
 import org.jetbrains.kotlin.js.backend.JsToStringGenerationVisitor
-import org.jetbrains.kotlin.js.backend.ast.*
+import org.jetbrains.kotlin.js.backend.ast.JsBlock
+import org.jetbrains.kotlin.js.backend.ast.JsGlobalBlock
+import org.jetbrains.kotlin.js.backend.ast.JsNode
+import org.jetbrains.kotlin.js.backend.ast.JsProgram
 import org.jetbrains.kotlin.js.dce.Context.Node
 import org.jetbrains.kotlin.js.facade.SourceMapBuilderConsumer
 import org.jetbrains.kotlin.js.inline.util.collectDefinedNames
@@ -33,7 +36,6 @@ import org.jetbrains.kotlin.js.sourceMap.SourceFilePathResolver
 import org.jetbrains.kotlin.js.sourceMap.SourceMap3Builder
 import org.jetbrains.kotlin.js.util.TextOutputImpl
 import java.io.File
-import java.io.FileInputStream
 import java.io.InputStreamReader
 
 class DeadCodeElimination(private val logConsumer: (DCELogLevel, String) -> Unit) {
@@ -41,7 +43,6 @@ class DeadCodeElimination(private val logConsumer: (DCELogLevel, String) -> Unit
     private val reachableNames = mutableSetOf<String>()
 
     var reachableNodes = setOf<Node>()
-        get
         private set
 
     fun apply(root: JsNode) {
@@ -82,19 +83,19 @@ class DeadCodeElimination(private val logConsumer: (DCELogLevel, String) -> Unit
             var hasErrors = false
             val blocks = inputFiles.map { file ->
                 val block = JsGlobalBlock()
-                val code = File(file.path).readText()
+                val code = file.resource.reader().let { InputStreamReader(it, "UTF-8") }.use { it.readText() }
-                val statements = parse(code, Reporter(file.path, logConsumer), program.scope, file.path) ?: run {
+                val statements = parse(code, Reporter(file.resource.name, logConsumer), program.scope, file.resource.name) ?: run {
                     hasErrors = true
                     return@map block
                 }
-                val sourceMapParse = file.pathToSourceMap
+                val sourceMapParse = file.sourceMapResource
-                        ?.let { InputStreamReader(FileInputStream(it), "UTF-8") }
+                        ?.let { InputStreamReader(it.reader(), "UTF-8") }
                         ?.use { SourceMapParser.parse(it) }
                 when (sourceMapParse) {
                     is SourceMapError -> {
                         logConsumer(
                                 DCELogLevel.WARN,
-                                "Error parsing source map file ${file.pathToSourceMap}: ${sourceMapParse.message}")
+                                "Error parsing source map file ${file.sourceMapResource}: ${sourceMapParse.message}")
                     }
                     is SourceMapSuccess -> {
                         val sourceMap = sourceMapParse.value
@@ -130,7 +131,7 @@ class DeadCodeElimination(private val logConsumer: (DCELogLevel, String) -> Unit
                     writeText(textOutput.toString())
                 }
 
-                if (file.pathToSourceMap != null) {
+                if (file.sourceMapResource != null) {
                     sourceMapFile.writeText(sourceMapContent)
                 }
             }

js/js.dce/src/org/jetbrains/kotlin/js/dce/InputFile.kt

@@ -16,4 +16,4 @@
 
 package org.jetbrains.kotlin.js.dce
 
-class InputFile(val path: String, val pathToSourceMap: String?, val outputPath: String, val moduleName: String? = null)
+class InputFile(val resource: InputResource, val sourceMapResource: InputResource?, val outputPath: String, val moduleName: String? = null)

js/js.dce/src/org/jetbrains/kotlin/js/dce/InputResource.kt

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.js.dce
+
+import java.io.File
+import java.io.FileInputStream
+import java.io.InputStream
+import java.util.zip.ZipFile
+
+class InputResource(val name: String, val reader: () -> InputStream) {
+    companion object {
+        fun file(path: String): InputResource = InputResource(path) { FileInputStream(File(path)) }
+
+        fun zipFile(path: String, entryPath: String): InputResource = InputResource("$path!$entryPath") {
+            val zipFile = ZipFile(path)
+            zipFile.getInputStream(zipFile.getEntry(entryPath))
+        }
+    }
+}

js/js.tests/test/org/jetbrains/kotlin/js/test/AbstractDceTest.kt

@@ -19,13 +19,15 @@ package org.jetbrains.kotlin.js.test
 import junit.framework.TestCase
 import org.jetbrains.kotlin.js.dce.DeadCodeElimination
 import org.jetbrains.kotlin.js.dce.InputFile
+import org.jetbrains.kotlin.js.dce.InputResource
 import java.io.File
 
 abstract class AbstractDceTest : TestCase() {
     fun doTest(filePath: String) {
         val file = File(filePath)
         val fileContents = file.readText()
-        val inputFile = InputFile(filePath, null, File(pathToOutputDir, file.relativeTo(File(pathToTestDir)).path).path, "main")
+        val inputFile = InputFile(InputResource.file(filePath), null,
+                                  File(pathToOutputDir, file.relativeTo(File(pathToTestDir)).path).path, "main")
         val dceResult = DeadCodeElimination.run(setOf(inputFile), extractDeclarations(REQUEST_REACHABLE_PATTERN, fileContents)) { _, _ -> }
         val reachableNodeStrings = dceResult.reachableNodes.map { it.toString().removePrefix("<unknown>.") }.toSet()
 

js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt

@@ -43,6 +43,7 @@ import org.jetbrains.kotlin.js.config.JsConfig
 import org.jetbrains.kotlin.js.config.SourceMapSourceEmbedding
 import org.jetbrains.kotlin.js.dce.DeadCodeElimination
 import org.jetbrains.kotlin.js.dce.InputFile
+import org.jetbrains.kotlin.js.dce.InputResource
 import org.jetbrains.kotlin.js.facade.*
 import org.jetbrains.kotlin.js.parser.parse
 import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapError
@@ -549,12 +550,12 @@ abstract class BasicBoxTest(
         val kotlinJsLibOutput = File(workDir, "kotlin.min.js").path
         val kotlinTestJsLibOutput = File(workDir, "kotlin-test.min.js").path
 
-        val kotlinJsInputFile = InputFile(kotlinJsLib, null, kotlinJsLibOutput, "kotlin")
+        val kotlinJsInputFile = InputFile(InputResource.file(kotlinJsLib), null, kotlinJsLibOutput, "kotlin")
-        val kotlinTestJsInputFile = InputFile(kotlinTestJsLib, null, kotlinTestJsLibOutput, "kotlin-test")
+        val kotlinTestJsInputFile = InputFile(InputResource.file(kotlinTestJsLib), null, kotlinTestJsLibOutput, "kotlin-test")
 
         val filesToMinify = generatedJsFiles.associate { (fileName, module) ->
             val inputFileName = File(fileName).nameWithoutExtension
-            fileName to InputFile(fileName, null, File(workDir, inputFileName + ".min.js").absolutePath, module.name)
+            fileName to InputFile(InputResource.file(fileName), null, File(workDir, inputFileName + ".min.js").absolutePath, module.name)
         }
 
         val testFunctionFqn = testModuleName + (if (testPackage.isNullOrEmpty()) "" else ".$testPackage") + ".$testFunction"

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinJsDcePlugin.kt

@@ -18,11 +18,8 @@ package org.jetbrains.kotlin.gradle.plugin
 
 import org.gradle.api.Plugin
 import org.gradle.api.Project
-import org.gradle.api.Task
-import org.gradle.api.internal.file.UnionFileCollection
 import org.gradle.api.internal.file.UnionFileTree
 import org.gradle.api.plugins.JavaPluginConvention
-import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.SourceSet
 import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
 import org.jetbrains.kotlin.gradle.tasks.KotlinJsDce
@@ -33,7 +30,7 @@ class KotlinJsDcePlugin : Plugin<Project> {
         project.pluginManager.apply(Kotlin2JsPluginWrapper::class.java)
 
         val javaPluginConvention = project.convention.getPlugin(JavaPluginConvention::class.java)
-        javaPluginConvention.sourceSets?.forEach { processSourceSet(project, it) }
+        javaPluginConvention.sourceSets.forEach { processSourceSet(project, it) }
     }
 
     private fun processSourceSet(project: Project, sourceSet: SourceSet) {
@@ -42,57 +39,26 @@ class KotlinJsDcePlugin : Plugin<Project> {
         val dceTaskName = sourceSet.getTaskName(DCE_TASK_PREFIX, TASK_SUFFIX)
         val dceTask = project.tasks.create(dceTaskName, KotlinJsDce::class.java).also {
             it.dependsOn(kotlinTask)
-            project.tasks.findByName("build")?.dependsOn(it)
+            project.tasks.findByName("build").dependsOn(it)
         }
 
         project.afterEvaluate {
             val outputDir = File(kotlinTask.outputFile).parentFile
 
-            val dependenciesDir = File(outputDir, "dependencies")
+            val configuration = project.configurations.findByName(sourceSet.compileConfigurationName)
-            val dependenciesTemporaryDir = File(outputDir, "dependencies-tmp")
+            val dceInputTrees = listOf(project.fileTree(kotlinTask.outputFile)) + configuration.map { project.fileTree(it) }
-            copyDependencies(project, sourceSet, dependenciesDir, dependenciesTemporaryDir, dceTask)
-
-            val dceInputTrees = listOf(project.fileTree(kotlinTask.outputFile), project.fileTree(dependenciesDir))
             val dceInputFiles = UnionFileTree("dce-input", dceInputTrees)
 
             with (dceTask) {
                 classpath = sourceSet.compileClasspath
                 destinationDir = File(outputDir, "min")
-                source(dceInputFiles.filter { it.path.endsWith(".js") })
+                source(dceInputFiles)
             }
         }
     }
 
-    private fun copyDependencies(project: Project, sourceSet: SourceSet, outputDir: File, tmpDir: File, dceTask: Task) {
-        val configuration = project.configurations.findByName(sourceSet.compileConfigurationName) ?: return
-
-        val zippedFiles = UnionFileCollection(configuration.map { project.zipTree(it) })
-        val files = project.fileTree(tmpDir)
-                .filter { file -> SUFFIXES.any { file.path.endsWith(it) } }
-                .filter { file -> SUFFIXES.any { File(file.path.removeSuffix(it) + ".meta.js").exists() } }
-
-        // This intermediate task is needed due to bug in Gradle that causes infinite loops in continuous build mode
-        val unpackName = sourceSet.getTaskName(UNPACK_DEPENDENCIES_TASK_PREFIX, TASK_SUFFIX)
-        val unpackTask = project.tasks.create(unpackName, Copy::class.java).apply {
-            from(zippedFiles)
-            into(tmpDir)
-        }
-
-        val name = sourceSet.getTaskName(DEPENDENCIES_TASK_PREFIX, TASK_SUFFIX)
-        with(project.tasks.create(name, Copy::class.java)) {
-            from(files)
-            into(outputDir)
-            includeEmptyDirs = true
-            dceTask.dependsOn(this)
-            dependsOn(unpackTask)
-        }
-    }
-
     companion object {
         private const val TASK_SUFFIX = "kotlinJs"
-        private const val UNPACK_DEPENDENCIES_TASK_PREFIX = "unpackDependencies"
-        private const val DEPENDENCIES_TASK_PREFIX = "copyDependencies"
         private const val DCE_TASK_PREFIX = "runDce"
-        private val SUFFIXES = listOf(".js", ".js.map")
     }
 }