From 811da2e28503d69f54085327610cd1fa25fbb3da Mon Sep 17 00:00:00 2001 From: Alexey Andreev Date: Fri, 16 Jun 2017 21:01:17 +0300 Subject: [PATCH] Compile stdlib-js with source map --- js/js.libraries/src/js/externs.js | 27 ---- .../{js/closure-wrapper.txt => wrapper.js} | 4 +- libraries/settings.gradle | 1 + libraries/stdlib/js/build.gradle | 77 +++++----- .../kotlin-stdlib-js-merger/build.gradle | 11 ++ .../kotlin-stdlib-js-merger/src/FileMerger.kt | 143 ++++++++++++++++++ 6 files changed, 197 insertions(+), 66 deletions(-) delete mode 100644 js/js.libraries/src/js/externs.js rename js/js.libraries/src/{js/closure-wrapper.txt => wrapper.js} (92%) create mode 100644 libraries/tools/kotlin-stdlib-js-merger/build.gradle create mode 100644 libraries/tools/kotlin-stdlib-js-merger/src/FileMerger.kt diff --git a/js/js.libraries/src/js/externs.js b/js/js.libraries/src/js/externs.js deleted file mode 100644 index 38d607bc7ab..00000000000 --- a/js/js.libraries/src/js/externs.js +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2010-2016 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Externs for closure compiler - -// from QUnit -function ok() {} - -// node.js -var module = {}; -var process = {}; - -// common -var console = {}; diff --git a/js/js.libraries/src/js/closure-wrapper.txt b/js/js.libraries/src/wrapper.js similarity index 92% rename from js/js.libraries/src/js/closure-wrapper.txt rename to js/js.libraries/src/wrapper.js index f121928f0b5..41b284acc64 100644 --- a/js/js.libraries/src/js/closure-wrapper.txt +++ b/js/js.libraries/src/wrapper.js @@ -12,5 +12,5 @@ }(this, function (Kotlin) { var _ = Kotlin; -%output% -})); \ No newline at end of file + insertContent(); +})); diff --git a/libraries/settings.gradle b/libraries/settings.gradle index ac5b4507001..c1aa9272efe 100644 --- a/libraries/settings.gradle +++ b/libraries/settings.gradle @@ -13,6 +13,7 @@ include ':kotlin-stdlib-jre7' include ':kotlin-stdlib-jre8' include ':kotlin-stdlib:samples' include ':tools:binary-compatibility-validator' +include ':tools:kotlin-stdlib-js-merger' include ':tools:kotlin-stdlib-gen' include ':tools:protobuf-lite' include ':kotlin-reflect' diff --git a/libraries/stdlib/js/build.gradle b/libraries/stdlib/js/build.gradle index 1372f56bcb4..6b4fe8458be 100644 --- a/libraries/stdlib/js/build.gradle +++ b/libraries/stdlib/js/build.gradle @@ -49,8 +49,12 @@ sourceSets { } } +configurations { + merger +} dependencies { testCompile project(':kotlin-test:kotlin-test-js') + merger project(":tools:kotlin-stdlib-js-merger") } createPreprocessorTask(project, "Main", "${projectDir}/../src/kotlin", commonSrcDir) @@ -110,6 +114,7 @@ compileBuiltinsKotlin2Js { kotlinOptions { metaInfo = false outputFile = "${buildDir}/classes/builtins/kotlin.js" + sourceMap = true } } @@ -117,6 +122,7 @@ compileKotlin2Js { dependsOn preprocessSourcesMain, prepareBuiltinsSources kotlinOptions { outputFile = "${buildDir}/classes/main/kotlin.js" + sourceMap = true } } @@ -127,51 +133,24 @@ compileTestKotlin2Js { } } -configurations { - closureCompiler -} -dependencies { - closureCompiler "com.google.javascript:closure-compiler:v20160315" -} - -task compileJs { +task compileJs(type: JavaExec) { dependsOn compileBuiltinsKotlin2Js, compileKotlin2Js inputs.files(compileBuiltinsKotlin2Js.outputs.files) inputs.files(compileKotlin2Js.outputs.files) inputs.dir(jsSrcDir) outputs.file(jsOutputFile) + outputs.file("${jsOutputFile}.map") + def inputFiles = fileTree("${jsSrcJsDir}") { + include '**/*.js' + } + + main = "org.jetbrains.kotlin.js.FileMergerKt" + args = [jsOutputFile, "$rootDir/../", "$jsSrcDir/wrapper.js"] + inputFiles.collect { it.path } + + ["$buildDir/classes/builtins/kotlin.js", "$buildDir/classes/main/kotlin.js"] + classpath = configurations.merger doLast { - ant.taskdef( - name: 'closureCompiler', - classname: 'com.google.javascript.jscomp.ant.CompileTask', - classpath: configurations.closureCompiler.asPath) - - ant.closureCompiler( - compilationLevel: 'whitespace', - prettyprint: 'true', - languagein: 'ECMASCRIPT5_STRICT', - warning: 'default', - output: jsOutputFile, - outputWrapperFile: "${jsSrcJsDir}/closure-wrapper.txt" - ) { - ant.path { - fileset(dir: "${jsSrcJsDir}") { - include(name: '**/*.js') - exclude(name: 'externs.js') - } - fileset(dir: "${buildDir}/classes") { - include(name: 'builtins/kotlin.js') - include(name: 'main/kotlin.js') - } - } - - externs(dir: "${jsSrcJsDir}") { - ant.file(name: 'externs.js') - } - } - ant.replaceregexp( file: jsOutputFile, match: "module.exports,\\s*require\\([^)]+\\)", @@ -187,6 +166,29 @@ task compileJs { match: "return\\s+_;", replace: "", byline: "true", encoding: "UTF-8") + + def sourceMapFile = file("${jsOutputFile}.map") + def sourceMap = new groovy.json.JsonSlurper().parseText(sourceMapFile.text) + def sourceMapSourcesBaseDirs = [ "${rootDir}/..", jsSrcJsDir ] + + sourceMap.sourcesContent = sourceMap.sources.collect { sourceName -> + sourceMapSourcesBaseDirs.collect { file("$it/$sourceName") }.find { it.exists() && it.name != "kotlin.js" }?.text + } + + def sourceMapBasePaths = [ + "js/js.libraries/src/", + "libraries/stdlib/js/build/builtin-sources/", + "libraries/stdlib/js/build/builtin-sources-for-builtins/", + "libraries/stdlib/js/build/builtin-sources-for-builtins/", + "libraries/stdlib/js/build/common-sources/" + ] + + sourceMap.sources = sourceMap.sources.collect { sourcePath -> + def prefixToRemove = sourceMapBasePaths.find { basePath -> sourcePath.startsWith(basePath) } + if (prefixToRemove != null) sourcePath.substring(prefixToRemove.length()) else sourcePath + } + + sourceMapFile.text = groovy.json.JsonOutput.toJson(sourceMap) } } @@ -211,6 +213,7 @@ task mergedJar(type: Jar, dependsOn: classes) { includeEmptyDirs false duplicatesStrategy DuplicatesStrategy.EXCLUDE from jsOutputFile + from "${jsOutputFile}.map" from sourceSets.main.output } diff --git a/libraries/tools/kotlin-stdlib-js-merger/build.gradle b/libraries/tools/kotlin-stdlib-js-merger/build.gradle new file mode 100644 index 00000000000..ace09028ed4 --- /dev/null +++ b/libraries/tools/kotlin-stdlib-js-merger/build.gradle @@ -0,0 +1,11 @@ +description = 'Merge utility for Kotlin Standard Library for JS' + +apply plugin: 'kotlin' + +dependencies { + compile files("$distDir/kotlin-compiler-for-maven.jar") +} + +sourceSets { + main.kotlin.srcDirs += 'src' +} diff --git a/libraries/tools/kotlin-stdlib-js-merger/src/FileMerger.kt b/libraries/tools/kotlin-stdlib-js-merger/src/FileMerger.kt new file mode 100644 index 00000000000..c479155556e --- /dev/null +++ b/libraries/tools/kotlin-stdlib-js-merger/src/FileMerger.kt @@ -0,0 +1,143 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.js + +import com.google.gwt.dev.js.ThrowExceptionOnErrorReporter +import org.jetbrains.kotlin.js.backend.JsToStringGenerationVisitor +import org.jetbrains.kotlin.js.backend.ast.* +import org.jetbrains.kotlin.js.facade.SourceMapBuilderConsumer +import org.jetbrains.kotlin.js.inline.util.fixForwardNameReferences +import org.jetbrains.kotlin.js.parser.parse +import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapError +import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapLocationRemapper +import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapParser +import org.jetbrains.kotlin.js.parser.sourcemaps.SourceMapSuccess +import org.jetbrains.kotlin.js.sourceMap.SourceFilePathResolver +import org.jetbrains.kotlin.js.sourceMap.SourceMap3Builder +import org.jetbrains.kotlin.js.util.TextOutputImpl +import org.json.JSONArray +import org.json.JSONObject +import org.json.JSONTokener +import java.io.File +import java.io.StringReader + +fun main(args: Array) { + val program = JsProgram() + + val outputFile = File(args[0]) + val baseDir = File(args[1]).canonicalFile.toPath() + fun File.relativizeIfNecessary(): String = baseDir.relativize(canonicalFile.toPath()).toString() + + val wrapperFile = File(args[2]) + val wrapper = parse(wrapperFile.readText(), ThrowExceptionOnErrorReporter, program.scope, wrapperFile.relativizeIfNecessary()) + val insertionPlace = wrapper.createInsertionPlace() + + val allFiles = mutableListOf() + args.drop(3).map { File(it) }.forEach { collectFiles(it, allFiles) } + + for (file in allFiles) { + val statements = parse(file.readText(), ThrowExceptionOnErrorReporter, program.scope, file.relativizeIfNecessary()) + val block = JsBlock(statements) + block.fixForwardNameReferences() + + val sourceMapFile = File(file.parent, file.name + ".map") + if (sourceMapFile.exists()) { + val sourceMapParse = sourceMapFile.reader().use { SourceMapParser.parse(it) } + when (sourceMapParse) { + is SourceMapError -> { + System.err.println("Error parsing source map file $sourceMapFile: ${sourceMapParse.message}") + System.exit(1) + } + is SourceMapSuccess -> { + val sourceMap = sourceMapParse.value + val remapper = SourceMapLocationRemapper(sourceMap) + remapper.remap(block) + } + } + } + + insertionPlace.statements += statements + } + + program.globalBlock.statements += wrapper + + val sourceMapFile = File(outputFile.parentFile, outputFile.name + ".map") + val textOutput = TextOutputImpl() + val sourceMapBuilder = SourceMap3Builder(outputFile, textOutput, "") + val consumer = SourceMapBuilderConsumer(sourceMapBuilder, SourceFilePathResolver(mutableListOf()), true, true) + program.globalBlock.accept(JsToStringGenerationVisitor(textOutput, consumer)) + val sourceMapContent = sourceMapBuilder.build() + + val programText = textOutput.toString() + + outputFile.writeText(programText + "\n//# sourceMappingURL=${sourceMapFile.name}\n") + + val sourceMapJson = StringReader(sourceMapContent).use { JSONObject(JSONTokener(it)) } + val sources = sourceMapJson["sources"] as JSONArray + + sourceMapJson.put("sourcesContent", sources.map { sourcePath -> + val sourceFile = File(sourcePath as String) + if (sourceFile.exists()) { + sourceFile.readText() + } + else { + null + } + }) + + sourceMapFile.writeText(sourceMapJson.toString(2)) +} + +private fun List.createInsertionPlace(): JsBlock { + val block = JsGlobalBlock() + + val visitor = object : JsVisitorWithContextImpl() { + override fun visit(x: JsExpressionStatement, ctx: JsContext): Boolean { + if (isInsertionPlace(x.expression)) { + ctx.replaceMe(block) + return false + } + else { + return super.visit(x, ctx) + } + } + + private fun isInsertionPlace(expression: JsExpression): Boolean { + if (expression !is JsInvocation || !expression.arguments.isEmpty()) return false + + val qualifier = expression.qualifier + if (qualifier !is JsNameRef || qualifier.qualifier != null) return false + return qualifier.ident == "insertContent" + } + } + + for (statement in this) { + visitor.accept(statement) + } + return block +} + +private fun collectFiles(rootFile: File, target: MutableList) { + if (rootFile.isDirectory) { + for (child in rootFile.listFiles().sorted()) { + collectFiles(child, target) + } + } + else if (rootFile.extension == "js") { + target += rootFile + } +} \ No newline at end of file