Compile stdlib-js with source map

js/js.libraries/src/js/externs.js

@@ -1,27 +0,0 @@
-/*
- * Copyright 2010-2016 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Externs for closure compiler
-
-// from QUnit
-function ok() {}
-
-// node.js
-var module = {};
-var process = {};
-
-// common
-var console = {};

js/js.libraries/src/wrapper.js

@@ -12,5 +12,5 @@
 }(this, function (Kotlin) {
     var _ = Kotlin;
 
-%output%
+    insertContent();
-}));
+}));

libraries/settings.gradle

@@ -13,6 +13,7 @@ include ':kotlin-stdlib-jre7'
 include ':kotlin-stdlib-jre8'
 include ':kotlin-stdlib:samples'
 include ':tools:binary-compatibility-validator'
+include ':tools:kotlin-stdlib-js-merger'
 include ':tools:kotlin-stdlib-gen'
 include ':tools:protobuf-lite'
 include ':kotlin-reflect'

libraries/stdlib/js/build.gradle

@@ -49,8 +49,12 @@ sourceSets {
     }
 }
 
+configurations {
+    merger
+}
 dependencies {
     testCompile project(':kotlin-test:kotlin-test-js')
+    merger project(":tools:kotlin-stdlib-js-merger")
 }
 
 createPreprocessorTask(project, "Main", "${projectDir}/../src/kotlin", commonSrcDir)
@@ -110,6 +114,7 @@ compileBuiltinsKotlin2Js {
     kotlinOptions {
         metaInfo = false
         outputFile = "${buildDir}/classes/builtins/kotlin.js"
+        sourceMap = true
     }
 }
 
@@ -117,6 +122,7 @@ compileKotlin2Js {
     dependsOn preprocessSourcesMain, prepareBuiltinsSources
     kotlinOptions {
         outputFile = "${buildDir}/classes/main/kotlin.js"
+        sourceMap = true
     }
 }
 
@@ -127,51 +133,24 @@ compileTestKotlin2Js {
     }
 }
 
-configurations {
+task compileJs(type: JavaExec) {
-    closureCompiler
-}
-dependencies {
-    closureCompiler "com.google.javascript:closure-compiler:v20160315"
-}
-
-task compileJs {
     dependsOn compileBuiltinsKotlin2Js, compileKotlin2Js
     inputs.files(compileBuiltinsKotlin2Js.outputs.files)
     inputs.files(compileKotlin2Js.outputs.files)
     inputs.dir(jsSrcDir)
     outputs.file(jsOutputFile)
+    outputs.file("${jsOutputFile}.map")
 
+    def inputFiles = fileTree("${jsSrcJsDir}") {
+        include '**/*.js'
+    }
+
+    main = "org.jetbrains.kotlin.js.FileMergerKt"
+    args = [jsOutputFile, "$rootDir/../", "$jsSrcDir/wrapper.js"] + inputFiles.collect { it.path } +
+           ["$buildDir/classes/builtins/kotlin.js", "$buildDir/classes/main/kotlin.js"]
+    classpath = configurations.merger
 
     doLast {
-        ant.taskdef(
-                name: 'closureCompiler',
-                classname: 'com.google.javascript.jscomp.ant.CompileTask',
-                classpath: configurations.closureCompiler.asPath)
-
-        ant.closureCompiler(
-                compilationLevel: 'whitespace',
-                prettyprint: 'true',
-                languagein: 'ECMASCRIPT5_STRICT',
-                warning: 'default',
-                output: jsOutputFile,
-                outputWrapperFile: "${jsSrcJsDir}/closure-wrapper.txt"
-        ) {
-            ant.path {
-                fileset(dir: "${jsSrcJsDir}") {
-                    include(name: '**/*.js')
-                    exclude(name: 'externs.js')
-                }
-                fileset(dir: "${buildDir}/classes") {
-                    include(name: 'builtins/kotlin.js')
-                    include(name: 'main/kotlin.js')
-                }
-            }
-
-            externs(dir: "${jsSrcJsDir}") {
-                ant.file(name: 'externs.js')
-            }
-        }
-
         ant.replaceregexp(
                 file: jsOutputFile,
                 match: "module.exports,\\s*require\\([^)]+\\)",
@@ -187,6 +166,29 @@ task compileJs {
                 match: "return\\s+_;",
                 replace: "",
                 byline: "true", encoding: "UTF-8")
+
+        def sourceMapFile = file("${jsOutputFile}.map")
+        def sourceMap = new groovy.json.JsonSlurper().parseText(sourceMapFile.text)
+        def sourceMapSourcesBaseDirs = [ "${rootDir}/..", jsSrcJsDir ]
+
+        sourceMap.sourcesContent = sourceMap.sources.collect { sourceName ->
+            sourceMapSourcesBaseDirs.collect { file("$it/$sourceName") }.find { it.exists() && it.name != "kotlin.js" }?.text
+        }
+
+        def sourceMapBasePaths = [
+                "js/js.libraries/src/",
+                "libraries/stdlib/js/build/builtin-sources/",
+                "libraries/stdlib/js/build/builtin-sources-for-builtins/",
+                "libraries/stdlib/js/build/builtin-sources-for-builtins/",
+                "libraries/stdlib/js/build/common-sources/"
+        ]
+
+        sourceMap.sources = sourceMap.sources.collect { sourcePath ->
+            def prefixToRemove = sourceMapBasePaths.find { basePath -> sourcePath.startsWith(basePath) }
+            if (prefixToRemove != null) sourcePath.substring(prefixToRemove.length()) else sourcePath
+        }
+
+        sourceMapFile.text = groovy.json.JsonOutput.toJson(sourceMap)
     }
 }
 
@@ -211,6 +213,7 @@ task mergedJar(type: Jar, dependsOn: classes) {
     includeEmptyDirs false
     duplicatesStrategy DuplicatesStrategy.EXCLUDE
     from jsOutputFile
+    from "${jsOutputFile}.map"
     from sourceSets.main.output
 }
 

libraries/tools/kotlin-stdlib-js-merger/build.gradle

@@ -0,0 +1,11 @@
+description = 'Merge utility for Kotlin Standard Library for JS'
+
+apply plugin: 'kotlin'
+
+dependencies {
+    compile files("$distDir/kotlin-compiler-for-maven.jar")
+}
+
+sourceSets {
+    main.kotlin.srcDirs += 'src'
+}

libraries/tools/kotlin-stdlib-js-merger/src/FileMerger.kt

@@ -0,0 +1,143 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.js
+
+import com.google.gwt.dev.js.ThrowExceptionOnErrorReporter
+import org.jetbrains.kotlin.js.backend.JsToStringGenerationVisitor
+import org.jetbrains.kotlin.js.backend.ast.*
+import org.jetbrains.kotlin.js.facade.SourceMapBuilderConsumer
+import org.jetbrains.kotlin.js.inline.util.fixForwardNameReferences
+import org.jetbrains.kotlin.js.parser.parse
+import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapError
+import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapLocationRemapper
+import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapParser
+import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapSuccess
+import org.jetbrains.kotlin.js.sourceMap.SourceFilePathResolver
+import org.jetbrains.kotlin.js.sourceMap.SourceMap3Builder
+import org.jetbrains.kotlin.js.util.TextOutputImpl
+import org.json.JSONArray
+import org.json.JSONObject
+import org.json.JSONTokener
+import java.io.File
+import java.io.StringReader
+
+fun main(args: Array<String>) {
+    val program = JsProgram()
+
+    val outputFile = File(args[0])
+    val baseDir = File(args[1]).canonicalFile.toPath()
+    fun File.relativizeIfNecessary(): String = baseDir.relativize(canonicalFile.toPath()).toString()
+
+    val wrapperFile = File(args[2])
+    val wrapper = parse(wrapperFile.readText(), ThrowExceptionOnErrorReporter, program.scope, wrapperFile.relativizeIfNecessary())
+    val insertionPlace = wrapper.createInsertionPlace()
+
+    val allFiles = mutableListOf<File>()
+    args.drop(3).map { File(it) }.forEach { collectFiles(it, allFiles) }
+
+    for (file in allFiles) {
+        val statements = parse(file.readText(), ThrowExceptionOnErrorReporter, program.scope, file.relativizeIfNecessary())
+        val block = JsBlock(statements)
+        block.fixForwardNameReferences()
+
+        val sourceMapFile = File(file.parent, file.name + ".map")
+        if (sourceMapFile.exists()) {
+            val sourceMapParse = sourceMapFile.reader().use { SourceMapParser.parse(it) }
+            when (sourceMapParse) {
+                is SourceMapError -> {
+                    System.err.println("Error parsing source map file $sourceMapFile: ${sourceMapParse.message}")
+                    System.exit(1)
+                }
+                is SourceMapSuccess -> {
+                    val sourceMap = sourceMapParse.value
+                    val remapper = SourceMapLocationRemapper(sourceMap)
+                    remapper.remap(block)
+                }
+            }
+        }
+
+        insertionPlace.statements += statements
+    }
+
+    program.globalBlock.statements += wrapper
+
+    val sourceMapFile = File(outputFile.parentFile, outputFile.name + ".map")
+    val textOutput = TextOutputImpl()
+    val sourceMapBuilder = SourceMap3Builder(outputFile, textOutput, "")
+    val consumer = SourceMapBuilderConsumer(sourceMapBuilder, SourceFilePathResolver(mutableListOf()), true, true)
+    program.globalBlock.accept(JsToStringGenerationVisitor(textOutput, consumer))
+    val sourceMapContent = sourceMapBuilder.build()
+
+    val programText = textOutput.toString()
+
+    outputFile.writeText(programText + "\n//# sourceMappingURL=${sourceMapFile.name}\n")
+
+    val sourceMapJson = StringReader(sourceMapContent).use { JSONObject(JSONTokener(it)) }
+    val sources = sourceMapJson["sources"] as JSONArray
+
+    sourceMapJson.put("sourcesContent", sources.map { sourcePath ->
+        val sourceFile = File(sourcePath as String)
+        if (sourceFile.exists()) {
+            sourceFile.readText()
+        }
+        else {
+            null
+        }
+    })
+
+    sourceMapFile.writeText(sourceMapJson.toString(2))
+}
+
+private fun List<JsStatement>.createInsertionPlace(): JsBlock {
+    val block = JsGlobalBlock()
+
+    val visitor = object : JsVisitorWithContextImpl() {
+        override fun visit(x: JsExpressionStatement, ctx: JsContext<in JsStatement>): Boolean {
+            if (isInsertionPlace(x.expression)) {
+                ctx.replaceMe(block)
+                return false
+            }
+            else {
+                return super.visit(x, ctx)
+            }
+        }
+
+        private fun isInsertionPlace(expression: JsExpression): Boolean {
+            if (expression !is JsInvocation || !expression.arguments.isEmpty()) return false
+
+            val qualifier = expression.qualifier
+            if (qualifier !is JsNameRef || qualifier.qualifier != null) return false
+            return qualifier.ident == "insertContent"
+        }
+    }
+
+    for (statement in this) {
+        visitor.accept(statement)
+    }
+    return block
+}
+
+private fun collectFiles(rootFile: File, target: MutableList<File>) {
+    if (rootFile.isDirectory) {
+        for (child in rootFile.listFiles().sorted()) {
+            collectFiles(child, target)
+        }
+    }
+    else if (rootFile.extension == "js") {
+        target += rootFile
+    }
+}