azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use source map remapper in JS inliner

Browse Source
This commit is contained in:
Alexey Andreev
2017-05-23 19:50:38 +03:00
parent 9c4ec902b0
commit bf21cfd6e0
14 changed files with 344 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
js/js.inliner/src/org/jetbrains/kotlin/js/inline/FunctionReader.kt
+42 -7
View File
@@ -26,7 +26,9 @@ import org.jetbrains.kotlin.js.backend.ast.metadata.inlineStrategy
import org.jetbrains.kotlin.js.config.JsConfig
import org.jetbrains.kotlin.js.inline.util.IdentitySet
import org.jetbrains.kotlin.js.inline.util.isCallInvocation
import org.jetbrains.kotlin.js.parser.OffsetToSourceMapping
import org.jetbrains.kotlin.js.parser.parseFunction
import org.jetbrains.kotlin.js.parser.sourcemaps.*
import org.jetbrains.kotlin.js.translate.context.Namer
import org.jetbrains.kotlin.js.translate.utils.JsDescriptorUtils.getModuleName
import org.jetbrains.kotlin.resolve.descriptorUtil.isExtension
@@ -34,6 +36,7 @@ import org.jetbrains.kotlin.resolve.inline.InlineStrategy
import org.jetbrains.kotlin.utils.JsLibraryUtils
import org.jetbrains.kotlin.utils.sure
import java.io.File
import java.io.StringReader
// TODO: add hash checksum to defineModule?
/**
@@ -57,7 +60,14 @@ class FunctionReader(private val config: JsConfig, private val currentModuleName
* kotlinVariable: kotlin object variable.
* The default variable is Kotlin, but it can be renamed by minifier.
*/
data class ModuleInfo(val filePath: String, val fileContent: String, val moduleVariable: String, val kotlinVariable: String)
class ModuleInfo(
val filePath: String,
val fileContent: String,
val moduleVariable: String,
val kotlinVariable: String,
val offsetToSourceMapping: OffsetToSourceMapping,
val sourceMap: SourceMap?
)
private val moduleNameToInfo = HashMultimap.create<String, ModuleInfo>()
@@ -68,22 +78,40 @@ class FunctionReader(private val config: JsConfig, private val currentModuleName
moduleNameMap = buildModuleNameMap(fragments)
JsLibraryUtils.traverseJsLibraries(libs) { fileContent, filePath ->
JsLibraryUtils.traverseJsLibraries(libs) { (content, path, sourceMapContent) ->
var current = 0
while (true) {
var index = fileContent.indexOf(DEFINE_MODULE_FIND_PATTERN, current)
var index = content.indexOf(DEFINE_MODULE_FIND_PATTERN, current)
if (index < 0) break
current = index + 1
index = rewindToIdentifierStart(fileContent, index)
val preciseMatcher = DEFINE_MODULE_PATTERN.matcher(offset(fileContent, index))
index = rewindToIdentifierStart(content, index)
val preciseMatcher = DEFINE_MODULE_PATTERN.matcher(offset(content, index))
if (!preciseMatcher.lookingAt()) continue
val moduleName = preciseMatcher.group(3)
val moduleVariable = preciseMatcher.group(4)
val kotlinVariable = preciseMatcher.group(1)
moduleNameToInfo.put(moduleName, ModuleInfo(filePath, fileContent, moduleVariable, kotlinVariable))
val sourceMap = sourceMapContent?.let {
val result = SourceMapParser.parse(StringReader(it))
when (result) {
is SourceMapSuccess -> result.value
is SourceMapError -> throw RuntimeException("Error parsing source map file: ${result.message}\n$it")
}
}
val moduleInfo = ModuleInfo(
filePath = path,
fileContent = content,
moduleVariable = moduleVariable,
kotlinVariable = kotlinVariable,
offsetToSourceMapping = OffsetToSourceMapping(content),
sourceMap = sourceMap
)
moduleNameToInfo.put(moduleName, moduleInfo)
}
}
}
@@ -152,9 +180,16 @@ class FunctionReader(private val config: JsConfig, private val currentModuleName
offset++
}
val function = parseFunction(source, info.filePath, offset, ThrowExceptionOnErrorReporter, JsRootScope(JsProgram()))
val position = info.offsetToSourceMapping[offset]
val function = parseFunction(source, info.filePath, position, offset, ThrowExceptionOnErrorReporter, JsRootScope(JsProgram()))
val moduleReference = moduleNameMap[tag] ?: currentModuleName.makeRef()
val sourceMap = info.sourceMap
if (sourceMap != null) {
val remapper = SourceMapLocationRemapper(mapOf(info.filePath to sourceMap))
remapper.remap(function)
}
val replacements = hashMapOf(info.moduleVariable to moduleReference,
info.kotlinVariable to Namer.kotlinObject())
replaceExternalNames(function, replacements)
js/js.parser/src/com/google/gwt/dev/js/rhino/LineBuffer.java
+3 -2
View File
@@ -64,9 +64,10 @@ final class LineBuffer {
*/
static final int BUFLEN = 256;
LineBuffer(Reader in, int lineno) {
LineBuffer(Reader in, CodePosition position) {
this.in = in;
this.lineno = lineno;
this.lineno = position.getLine();
this.lineStart = -position.getOffset();
}
int read() throws IOException {
js/js.parser/src/com/google/gwt/dev/js/rhino/TokenStream.java
+5 -4
View File
@@ -495,14 +495,15 @@ public class TokenStream {
}
public TokenStream(Reader in,
String sourceName, int lineno)
String sourceName, CodePosition position)
{
this.in = new LineBuffer(in, lineno);
this.in = new LineBuffer(in, position);
this.pushbackToken = EOF;
this.sourceName = sourceName;
flags = 0;
secondToLastPosition = new CodePosition(lineno, 0);
lastPosition = new CodePosition(lineno, 0);
secondToLastPosition = position;
lastPosition = position;
lastTokenPosition = position;
}
/* return and pop the token from the stream if it matches...
js/js.parser/src/org/jetbrains/kotlin/js/parser/OffsetToSourceMapping.kt
+37
View File
@@ -0,0 +1,37 @@
/*
* Copyright 2010-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.kotlin.js.parser
import com.google.gwt.dev.js.rhino.CodePosition
class OffsetToSourceMapping(text: String) {
private val data: IntArray
init {
val lineSeparators = LINE_SEPARATOR.findAll(text).map { it.range.endInclusive + 1 }
data = (sequenceOf(0) + lineSeparators).toList().toIntArray()
}
operator fun get(offset: Int): CodePosition {
val lineNumber = data.binarySearch(offset).let { if (it >= 0) it else -it - 2 }
return CodePosition(lineNumber, offset - data[lineNumber])
}
private companion object {
private val LINE_SEPARATOR = Regex("\\r\\n|\\r|\\n")
}
}
js/js.parser/src/org/jetbrains/kotlin/js/parser/parserUtils.kt
+8 -10
View File
@@ -16,29 +16,26 @@
package org.jetbrains.kotlin.js.parser
import com.google.gwt.dev.js.*
import com.google.gwt.dev.js.JsAstMapper
import com.google.gwt.dev.js.rhino.*
import org.jetbrains.kotlin.js.backend.ast.JsFunction
import org.jetbrains.kotlin.js.backend.ast.JsFunctionScope
import org.jetbrains.kotlin.js.backend.ast.JsScope
import org.jetbrains.kotlin.js.backend.ast.JsStatement
import org.jetbrains.kotlin.js.common.SourceInfoImpl
import java.io.*
import java.io.Reader
import java.io.StringReader
import java.util.*
private val FAKE_SOURCE_INFO = SourceInfoImpl(null, 0, 0, 0, 0)
fun parse(code: String, reporter: ErrorReporter, scope: JsScope, fileName: String): List<JsStatement> {
val insideFunction = scope is JsFunctionScope
val node = parse(code, 0, reporter, insideFunction, Parser::parse)
val node = parse(code, CodePosition(0, 0), 0, reporter, insideFunction, Parser::parse)
return node.toJsAst(scope, fileName) {
mapStatements(it)
}
}
fun parseFunction(code: String, fileName: String, offset: Int, reporter: ErrorReporter, scope: JsScope): JsFunction =
parse(code, offset, reporter, insideFunction = false) {
fun parseFunction(code: String, fileName: String, position: CodePosition, offset: Int, reporter: ErrorReporter, scope: JsScope): JsFunction =
parse(code, position, offset, reporter, insideFunction = false) {
addObserver(FunctionParsingObserver())
primaryExpr(it)
}.toJsAst(scope, fileName, JsAstMapper::mapFunction)
@@ -65,6 +62,7 @@ private class FunctionParsingObserver : Observer {
inline
private fun parse(
code: String,
startPosition: CodePosition,
offset: Int,
reporter: ErrorReporter,
insideFunction: Boolean,
@@ -73,7 +71,7 @@ private fun parse(
Context.enter().errorReporter = reporter
try {
val ts = TokenStream(StringReader(code, offset), "<parser>", FAKE_SOURCE_INFO.line)
val ts = TokenStream(StringReader(code, offset), "<parser>", startPosition)
val parser = Parser(IRFactory(ts), insideFunction)
return parser.parseAction(ts) as Node
} finally {
js/js.tests/test/org/jetbrains/kotlin/js/test/AbstractJsLineNumberTest.kt
+116 -35
View File
@@ -39,61 +39,104 @@ import org.jetbrains.kotlin.js.test.utils.LineCollector
import org.jetbrains.kotlin.js.test.utils.LineOutputToStringVisitor
import org.jetbrains.kotlin.js.util.TextOutputImpl
import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.serialization.js.JsModuleDescriptor
import org.jetbrains.kotlin.serialization.js.KotlinJavascriptSerializationUtil
import org.jetbrains.kotlin.serialization.js.ModuleKind
import org.jetbrains.kotlin.test.KotlinTestUtils
import org.jetbrains.kotlin.test.KotlinTestWithEnvironment
import org.jetbrains.kotlin.utils.DFS
import java.io.ByteArrayOutputStream
import java.io.Closeable
import java.io.File
import java.io.PrintStream
import java.nio.charset.Charset
abstract class AbstractJsLineNumberTest : KotlinTestWithEnvironment() {
fun doTest(filePath: String) {
val translator = K2JSTranslator(createConfig())
val unit = TranslationUnit.SourceFile(createPsiFile(filePath))
val translationResult = translator.translateUnits(listOf(unit), MainCallParameters.noCall())
val file = File(filePath)
val sourceCode = file.readText()
if (translationResult !is TranslationResult.Success) {
val outputStream = ByteArrayOutputStream()
val collector = PrintingMessageCollector(PrintStream(outputStream), MessageRenderer.PLAIN_FULL_PATHS, true)
AnalyzerWithCompilerReport.reportDiagnostics(translationResult.diagnostics, collector)
val messages = outputStream.toByteArray().toString(Charset.forName("UTF-8"))
throw AssertionError("The following errors occurred compiling test:\n" + messages)
TestFileFactoryImpl().use { testFactory ->
val inputFiles = KotlinTestUtils.createTestFiles(file.name, sourceCode, testFactory, true)
val modules = inputFiles
.map { it.module }.distinct()
.associateBy { it.name }
val orderedModules = DFS.topologicalOrder(modules.values) { module -> module.dependencies.mapNotNull { modules[it] } }
orderedModules.asReversed().forEach { module ->
val baseOutputPath = module.outputFileName(file)
val translator = K2JSTranslator(createConfig(module, file, modules))
val units = module.files.map { TranslationUnit.SourceFile(createPsiFile(it.fileName)) }
val translationResult = translator.translateUnits(units, MainCallParameters.noCall())
if (translationResult !is TranslationResult.Success) {
val outputStream = ByteArrayOutputStream()
val collector = PrintingMessageCollector(PrintStream(outputStream), MessageRenderer.PLAIN_FULL_PATHS, true)
AnalyzerWithCompilerReport.reportDiagnostics(translationResult.diagnostics, collector)
val messages = outputStream.toByteArray().toString(Charset.forName("UTF-8"))
throw AssertionError("The following errors occurred compiling test:\n" + messages)
}
val lineCollector = LineCollector()
lineCollector.accept(translationResult.program)
val programOutput = TextOutputImpl()
translationResult.program.globalBlock.accept(LineOutputToStringVisitor(programOutput, lineCollector))
val generatedCode = programOutput.toString()
with(File(baseOutputPath + "-lines.js")) {
parentFile.mkdirs()
writeText(generatedCode)
}
File(baseOutputPath + ".js").writeText(translationResult.program.globalBlock.toString())
val moduleDescription = JsModuleDescriptor(
name = module.name,
data = translationResult.moduleDescriptor,
kind = ModuleKind.PLAIN,
imported = emptyList()
)
val metaFileContent = KotlinJavascriptSerializationUtil.metadataAsString(
translationResult.bindingContext, moduleDescription)
File(baseOutputPath + ".meta.js").writeText(metaFileContent)
val linesMatcher = module.files
.mapNotNull { LINES_PATTERN.find(File(it.fileName).readText()) }
.firstOrNull()
?: error("'// LINES: ' comment was not found in source file. Generated code is:\n$generatedCode")
val expectedLines = linesMatcher.groups[1]!!.value
val actualLines = lineCollector.lines
.dropLastWhile { it == null }
.joinToString(" ") { if (it == null) "*" else (it + 1).toString() }
TestCase.assertEquals(generatedCode, expectedLines, actualLines)
}
}
val lineCollector = LineCollector()
lineCollector.accept(translationResult.program)
val programOutput = TextOutputImpl()
translationResult.program.globalBlock.accept(LineOutputToStringVisitor(programOutput, lineCollector))
val generatedCode = programOutput.toString()
val relativePath = File(filePath).relativeTo(File(BASE_PATH)).path.removeSuffix(".kt")
with(File(File(OUT_PATH), relativePath + ".js")) {
parentFile.mkdirs()
writeText(generatedCode)
}
val sourceCode = FileUtil.loadFile(File(filePath))
val linesMatcher = LINES_PATTERN.find(sourceCode) ?:
error("'// LINES: ' comment was not found in source file. Generated code is:\n$generatedCode")
val expectedLines = linesMatcher.groups[1]!!.value
val actualLines = lineCollector.lines
.dropLastWhile { it == null }
.joinToString(" ") { if (it == null) "*" else (it + 1).toString() }
TestCase.assertEquals(generatedCode, expectedLines, actualLines)
}
private fun TestModule.outputFileName(file: File): String {
return outputPath(file) + "-" + name
}
private fun outputPath(file: File) = File(OUT_PATH, file.relativeTo(File(BASE_PATH)).path.removeSuffix(".kt")).path
override fun createEnvironment(): KotlinCoreEnvironment {
return KotlinCoreEnvironment.createForTests(testRootDisposable, CompilerConfiguration(), EnvironmentConfigFiles.JS_CONFIG_FILES)
}
private fun createConfig(): JsConfig {
private fun createConfig(module: TestModule, inputFile: File, modules: Map<String, TestModule>): JsConfig {
val dependencies = module.dependencies
.mapNotNull { modules[it]?.outputFileName(inputFile) }
.map { "$it.meta.js" }
val configuration = environment.configuration.copy()
configuration.put(JSConfigurationKeys.LIBRARIES, JsConfig.JS_STDLIB + JsConfig.JS_KOTLIN_TEST)
configuration.put(JSConfigurationKeys.LIBRARIES, JsConfig.JS_STDLIB + JsConfig.JS_KOTLIN_TEST + dependencies )
configuration.put(CommonConfigurationKeys.MODULE_NAME, "test")
configuration.put(CommonConfigurationKeys.MODULE_NAME, module.name)
configuration.put(JSConfigurationKeys.MODULE_KIND, ModuleKind.PLAIN)
configuration.put(JSConfigurationKeys.TARGET, EcmaVersion.v5)
@@ -109,6 +152,44 @@ abstract class AbstractJsLineNumberTest : KotlinTestWithEnvironment() {
return psiManager.findFile(fileSystem.findFileByPath(fileName)!!) as KtFile
}
private inner class TestFileFactoryImpl : KotlinTestUtils.TestFileFactory<TestModule, TestFile>, Closeable {
val tmpDir = KotlinTestUtils.tmpDir("js-tests")
val defaultModule = TestModule(BasicBoxTest.TEST_MODULE, emptyList())
override fun createFile(module: TestModule?, fileName: String, text: String, directives: Map<String, String>): TestFile? {
val currentModule = module ?: defaultModule
val temporaryFile = File(tmpDir, "${currentModule.name}/$fileName")
KotlinTestUtils.mkdirs(temporaryFile.parentFile)
temporaryFile.writeText(text, Charsets.UTF_8)
return TestFile(temporaryFile.absolutePath, currentModule)
}
override fun createModule(name: String, dependencies: List<String>, friends: List<String>): TestModule? {
return TestModule(name, dependencies)
}
override fun close() {
FileUtil.delete(tmpDir)
}
}
private class TestModule(
val name: String,
dependencies: List<String>
) {
val dependencies = dependencies.toMutableList()
val files = mutableListOf<TestFile>()
}
private class TestFile(val fileName: String, val module: TestModule) {
init {
module.files += this
}
}
companion object {
private val DIR_NAME = "lineNumbers"
private val LINES_PATTERN = Regex("^ *// *LINES: *(.*)$", RegexOption.MULTILINE)
js/js.tests/test/org/jetbrains/kotlin/js/test/BasicBoxTest.kt
+2 -4
View File
@@ -87,9 +87,7 @@ abstract class BasicBoxTest(
val outputPostfixFile = getOutputPostfixFile(filePath)
TestFileFactoryImpl().use { testFactory ->
testFactory.defaultModule.moduleKind
val inputFiles = KotlinTestUtils.createTestFiles(file.name, fileContent, testFactory)
val inputFiles = KotlinTestUtils.createTestFiles(file.name, fileContent, testFactory, true)
val modules = inputFiles
.map { it.module }.distinct()
.map { it.name to it }.toMap()
@@ -627,7 +625,7 @@ abstract class BasicBoxTest(
private val METADATA_EXTENSION = "jsmeta"
private val HEADER_FILE = "header.$METADATA_EXTENSION"
private val TEST_MODULE = "JS_TESTS"
val TEST_MODULE = "JS_TESTS"
private val DEFAULT_MODULE = "main"
private val TEST_FUNCTION = "box"
private val OLD_MODULE_SUFFIX = "-old"
js/js.tests/test/org/jetbrains/kotlin/js/test/JsLineNumberTestGenerated.java
+15
View File
@@ -269,4 +269,19 @@ public class JsLineNumberTestGenerated extends AbstractJsLineNumberTest {
String fileName = KotlinTestUtils.navigationMetadata("js/js.translator/testData/lineNumbers/whileWithComplexCondition.kt");
doTest(fileName);
}
@TestMetadata("js/js.translator/testData/lineNumbers/inlineMultiModule")
@TestDataPath("$PROJECT_ROOT")
@RunWith(JUnit3RunnerWithInners.class)
public static class InlineMultiModule extends AbstractJsLineNumberTest {
public void testAllFilesPresentInInlineMultiModule() throws Exception {
KotlinTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("js/js.translator/testData/lineNumbers/inlineMultiModule"), Pattern.compile("^([^_](.+))\\.kt$"), TargetBackend.JS, true);
}
@TestMetadata("simple.kt")
public void testSimple() throws Exception {
String fileName = KotlinTestUtils.navigationMetadata("js/js.translator/testData/lineNumbers/inlineMultiModule/simple.kt");
doTest(fileName);
}
}
}
js/js.translator/src/org/jetbrains/kotlin/js/facade/TranslationResult.kt
+2 -2
View File
@@ -47,8 +47,8 @@ abstract class TranslationResult protected constructor(val diagnostics: Diagnost
val program: JsProgram,
diagnostics: Diagnostics,
private val importedModules: List<String>,
private val moduleDescriptor: ModuleDescriptor,
private val bindingContext: BindingContext,
val moduleDescriptor: ModuleDescriptor,
val bindingContext: BindingContext,
val metadataHeader: ByteArray?,
val fileTranslationResults: Map<KtFile, FileTranslationResult>
) : TranslationResult(diagnostics) {
js/js.translator/testData/lineNumbers/delegateMemberVal.kt
Vendored
+8 -2
View File
@@ -1,10 +1,16 @@
class A {
val z by
lazy { 23 }
Delegate { 23 }
}
fun foo() {
println(A().z)
}
// LINES: 2 3 * 2 * 3 * 7
class Delegate(val f: () -> Int) {
operator fun getValue(thisRef: Any?, property: Any): Int {
return f()
}
}
// LINES: 2 3 2 * 3 * 7 10 12
js/js.translator/testData/lineNumbers/inlineMultiModule/simple.kt
Vendored
+24
View File
@@ -0,0 +1,24 @@
// MODULE: lib
// FILE: lib.kt
package pkg1
inline fun foo(x: String) {
println("foo1($x);")
println("foo2($x);")
}
// LINES: 7 8
// MODULE: main(lib)
// FILE: main.kt
package pkg2
import pkg1.*
fun box() {
foo("23")
foo("42")
}
// LINES: 7 20 7 8 20 8 7 21 7 8 21 8