JS: add metadata to AST serialization/deserialization

js/js.ast/src/org/jetbrains/kotlin/js/backend/ast/JsLocation.kt

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.js.backend.ast
+
+data class JsLocation(
+        val file: String,
+        val startLine: Int,
+        val startChar: Int
+)

js/js.serializer/src/js-ast.proto

@@ -20,32 +20,49 @@ option java_outer_classname = "JsAstProtoBuf";
 option optimize_for = LITE_RUNTIME;
 
 
+message Location {
+    required int32 startLine = 1;
+    required int32 startChar = 2;
+}
+
+enum SideEffects {
+    AFFECTS_STATE = 1;
+    DEPENDS_ON_STATE = 2;
+    PURE = 3;
+}
+
+
 // Expressions
 //
 
 message Expression {
+    optional int32 fileId = 1;
+    optional Location location = 2;
+    optional bool synthetic = 3 [default = false];
+    optional SideEffects side_effects = 4 [default = AFFECTS_STATE];
+
     oneof expression {
-        int32 simple_name_reference = 2;
+        int32 simple_name_reference = 22;
-        ThisLiteral this_literal = 3;
+        ThisLiteral this_literal = 23;
-        NullLiteral null_literal = 4;
+        NullLiteral null_literal = 24;
-        TrueLiteral true_literal = 5;
+        TrueLiteral true_literal = 25;
-        FalseLiteral false_literal = 6;
+        FalseLiteral false_literal = 26;
-        StringLiteral string_literal = 7;
+        StringLiteral string_literal = 27;
-        RegExpLiteral reg_exp_literal = 8;
+        RegExpLiteral reg_exp_literal = 28;
-        IntLiteral int_literal = 9;
+        IntLiteral int_literal = 29;
-        DoubleLiteral double_literal = 10;
+        DoubleLiteral double_literal = 30;
-        ArrayLiteral array_literal = 11;
+        ArrayLiteral array_literal = 31;
-        ObjectLiteral object_literal = 12;
+        ObjectLiteral object_literal = 32;
-        Function function = 13;
+        Function function = 33;
-        DocComment doc_comment = 14;
+        DocComment doc_comment = 34;
-        BinaryOperation binary = 15;
+        BinaryOperation binary = 35;
-        UnaryOperation unary = 16;
+        UnaryOperation unary = 36;
-        Conditional conditional = 17;
+        Conditional conditional = 37;
-        ArrayAccess array_access = 18;
+        ArrayAccess array_access = 38;
-        NameReference name_reference = 19;
+        NameReference name_reference = 39;
-        PropertyReference property_reference = 20;
+        PropertyReference property_reference = 40;
-        Invocation invocation = 21;
+        Invocation invocation = 41;
-        Instantiation instantiation = 22;
+        Instantiation instantiation = 42;
     }
 }
 
@@ -84,7 +101,7 @@ message ArrayLiteral {
 
 message ObjectLiteral {
     repeated ObjectLiteralEntry entry = 1;
-    required bool multiline = 2;
+    optional bool multiline = 2 [default = true];
 }
 
 message ObjectLiteralEntry {
@@ -96,10 +113,12 @@ message Function {
     repeated Parameter parameter = 1;
     optional int32 name_id = 2;
     required Statement body = 3;
+    optional bool local = 4 [default = false];
 }
 
 message Parameter {
     required int32 name_id = 1;
+    optional bool has_default_value = 2 [default = false];
 }
 
 message DocComment {
@@ -190,7 +209,7 @@ message ArrayAccess {
 
 message NameReference {
     required int32 name_id = 1;
-    required Expression qualifier = 2;
+    optional Expression qualifier = 2;
 }
 
 message PropertyReference {
@@ -213,25 +232,29 @@ message Instantiation {
 //
 
 message Statement {
+    optional int32 fileId = 1;
+    optional Location location = 2;
+    optional bool synthetic = 3 [default = false];
+
     oneof statement {
-        Return return_statement = 1;
+        Return return_statement = 21;
-        Throw throw_statement = 2;
+        Throw throw_statement = 22;
-        Break break_statement = 3;
+        Break break_statement = 23;
-        Continue continue_statement = 4;
+        Continue continue_statement = 24;
-        Debugger debugger = 5;
+        Debugger debugger = 25;
-        ExpressionStatement expression = 6;
+        ExpressionStatement expression = 26;
-        Vars vars = 7;
+        Vars vars = 27;
-        Block block = 8;
+        Block block = 28;
-        GlobalBlock global_block = 9;
+        GlobalBlock global_block = 29;
-        Label label = 10;
+        Label label = 30;
-        If if_statement = 11;
+        If if_statement = 31;
-        Switch switch_statement = 12;
+        Switch switch_statement = 32;
-        While while_statement = 13;
+        While while_statement = 33;
-        DoWhile do_while_statement = 14;
+        DoWhile do_while_statement = 34;
-        For for_statement = 15;
+        For for_statement = 35;
-        ForIn for_in_statement = 16;
+        ForIn for_in_statement = 36;
-        Try try_statement = 17;
+        Try try_statement = 37;
-        Empty empty = 18;
+        Empty empty = 38;
     }
 }
 
@@ -260,7 +283,8 @@ message ExpressionStatement {
 
 message Vars {
     repeated VarDeclaration declaration = 1;
-    required bool multiline = 2;
+    optional bool multiline = 2 [default = false];
+    optional int32 exported_package_id = 3;
 }
 
 message VarDeclaration {

js/js.serializer/src/org/jetbrains/kotlin/serialization/js/ast/JsAstDeserializer.kt

@@ -17,19 +17,23 @@
 package org.jetbrains.kotlin.serialization.js.ast
 
 import org.jetbrains.kotlin.js.backend.ast.*
+import org.jetbrains.kotlin.js.backend.ast.metadata.*
+import org.jetbrains.kotlin.protobuf.CodedInputStream
 import org.jetbrains.kotlin.serialization.js.ast.JsAstProtoBuf.*
 import org.jetbrains.kotlin.serialization.js.ast.JsAstProtoBuf.Expression.ExpressionCase
 import org.jetbrains.kotlin.serialization.js.ast.JsAstProtoBuf.Statement.StatementCase
 import java.io.InputStream
+import java.util.*
 
 class JsAstDeserializer(private val program: JsProgram) {
     private val scope = JsRootScope(program)
     private val stringTable = mutableListOf<String>()
     private val nameTable = mutableListOf<Name>()
     private val nameCache = mutableListOf<JsName?>()
+    private val locationStack: Deque<JsLocation> = ArrayDeque()
 
     fun deserialize(input: InputStream): JsProgramFragment {
-        return deserialize(Chunk.parseFrom(input))
+        return deserialize(Chunk.parseFrom(CodedInputStream.newInstance(input).apply { setRecursionLimit(4096) }))
     }
 
     fun deserialize(proto: Chunk): JsProgramFragment {
@@ -94,7 +98,19 @@ class JsAstDeserializer(private val program: JsProgram) {
         }
     }
 
-    private fun deserialize(proto: Statement): JsStatement = when (proto.statementCase) {
+    private fun deserialize(proto: Statement): JsStatement {
+        val statement = withLocation(
+                fileId = if (proto.hasFileId()) proto.fileId else null,
+                location = if (proto.hasLocation()) proto.location else null,
+                action = { deserializeNoMetadata(proto) }
+        )
+        if (statement is HasMetadata) {
+            statement.synthetic = proto.synthetic
+        }
+        return statement
+    }
+
+    private fun deserializeNoMetadata(proto: Statement): JsStatement = when (proto.statementCase) {
         StatementCase.RETURN_STATEMENT -> {
             val returnProto = proto.returnStatement
             JsReturn(if (returnProto.hasValue()) deserialize(returnProto.value) else null)
@@ -226,7 +242,18 @@ class JsAstDeserializer(private val program: JsProgram) {
         null -> error("Statement not set")
     }
 
-    private fun deserialize(proto: Expression): JsExpression = when(proto.expressionCase) {
+    private fun deserialize(proto: Expression): JsExpression {
+        val expression = withLocation(
+                fileId = if (proto.hasFileId()) proto.fileId else null,
+                location = if (proto.hasLocation()) proto.location else null,
+                action = { deserializeNoMetadata(proto) }
+        )
+        expression.synthetic = proto.synthetic
+        expression.sideEffects = map(proto.sideEffects)
+        return expression
+    }
+
+    private fun deserializeNoMetadata(proto: Expression): JsExpression = when (proto.expressionCase) {
         ExpressionCase.THIS_LITERAL -> JsLiteral.THIS
         ExpressionCase.NULL_LITERAL -> JsLiteral.NULL
         ExpressionCase.TRUE_LITERAL -> JsLiteral.TRUE
@@ -268,6 +295,7 @@ class JsAstDeserializer(private val program: JsProgram) {
                 if (functionProto.hasNameId()) {
                     name = deserializeName(functionProto.nameId)
                 }
+                isLocal = functionProto.local
             }
         }
 
@@ -348,6 +376,9 @@ class JsAstDeserializer(private val program: JsProgram) {
             val initialValue = if (declProto.hasInitialValue()) deserialize(declProto.initialValue) else null
             vars.vars += JsVars.JsVar(deserializeName(declProto.nameId), initialValue)
         }
+        if (proto.hasExportedPackageId()) {
+            vars.exportedPackage = deserializeString(proto.exportedPackageId)
+        }
         return vars
     }
 
@@ -356,7 +387,9 @@ class JsAstDeserializer(private val program: JsProgram) {
     }
 
     private fun deserializeParameter(proto: Parameter): JsParameter {
-        return JsParameter(deserializeName(proto.nameId))
+        return JsParameter(deserializeName(proto.nameId)).apply {
+            hasDefaultValue = proto.hasDefaultValue
+        }
     }
 
     private fun deserializeName(id: Int): JsName {
@@ -426,4 +459,38 @@ class JsAstDeserializer(private val program: JsProgram) {
         UnaryOperation.Type.TYPEOF -> JsUnaryOperator.TYPEOF
         UnaryOperation.Type.VOID -> JsUnaryOperator.VOID
     }
+
+    private fun map(sideEffects: SideEffects) = when (sideEffects) {
+        SideEffects.AFFECTS_STATE -> SideEffectKind.AFFECTS_STATE
+        SideEffects.DEPENDS_ON_STATE -> SideEffectKind.DEPENDS_ON_STATE
+        SideEffects.PURE -> SideEffectKind.PURE
+    }
+
+    private fun <T : JsNode> withLocation(fileId: Int?, location: Location?, action: () -> T): T {
+        val lastLocation = locationStack.peek()
+        val deserializedLocation = if (fileId != null || location != null) {
+            val file = fileId?.let { deserializeString(it) } ?: lastLocation?.file!!
+            val (startLine, startChar) = if (location != null) {
+                Pair(location.startLine, location.startChar)
+            }
+            else {
+                Pair(lastLocation.startLine, lastLocation.startChar)
+            }
+            JsLocation(file, startLine, startChar)
+        }
+        else {
+            null
+        }
+
+        if (deserializedLocation != null) {
+            locationStack.push(deserializedLocation)
+        }
+        val node = action()
+        if (deserializedLocation != null) {
+            node.source = deserializedLocation
+            locationStack.pop()
+        }
+
+        return node
+    }
 }

js/js.serializer/src/org/jetbrains/kotlin/serialization/js/ast/JsAstSerializer.kt

@@ -16,17 +16,21 @@
 
 package org.jetbrains.kotlin.serialization.js.ast
 
+import com.intellij.psi.PsiElement
 import org.jetbrains.kotlin.js.backend.ast.*
+import org.jetbrains.kotlin.js.backend.ast.metadata.*
 import org.jetbrains.kotlin.serialization.js.ast.JsAstProtoBuf.*
 import org.jetbrains.kotlin.serialization.js.ast.JsAstProtoBuf.BinaryOperation.Type.*
 import org.jetbrains.kotlin.serialization.js.ast.JsAstProtoBuf.UnaryOperation.Type.*
 import java.io.OutputStream
+import java.util.*
 
 class JsAstSerializer {
     private val nameTableBuilder = NameTable.newBuilder()
     private val stringTableBuilder = StringTable.newBuilder()
     private val nameMap = mutableMapOf<JsName, Int>()
     private val stringMap = mutableMapOf<String, Int>()
+    private val locationStack: Deque<JsLocation> = ArrayDeque()
 
     fun serialize(fragment: JsProgramFragment, output: OutputStream) {
         serialize(fragment).writeTo(output)
@@ -246,7 +250,14 @@ class JsAstSerializer {
             }
         }
 
-        statement.accept(visitor)
+        withLocation(statement, { visitor.builder.fileId = it }, {visitor.builder.location = it }) {
+            statement.accept(visitor)
+        }
+
+        if (statement is HasMetadata && statement.synthetic) {
+            visitor.builder.synthetic = true
+        }
+
         return visitor.builder.build()
     }
 
@@ -319,6 +330,9 @@ class JsAstSerializer {
                 x.parameters.forEach { functionBuilder.addParameter(serializeParameter(it)) }
                 x.name?.let { functionBuilder.nameId = serialize(it) }
                 functionBuilder.body = serialize(x.body)
+                if (x.isLocal) {
+                    functionBuilder.local = true
+                }
                 builder.function = functionBuilder.build()
             }
 
@@ -404,13 +418,24 @@ class JsAstSerializer {
             }
         }
 
-        expression.accept(visitor)
+        withLocation(expression, { visitor.builder.fileId = it }, {visitor.builder.location = it }) {
+            expression.accept(visitor)
+        }
+
+        with (visitor.builder) {
+            synthetic = expression.synthetic
+            sideEffects = map(expression.sideEffects)
+        }
+
         return visitor.builder.build()
     }
 
     private fun serializeParameter(parameter: JsParameter): Parameter {
         val parameterBuilder = Parameter.newBuilder()
         parameterBuilder.nameId = serialize(parameter.name)
+        if (parameter.hasDefaultValue) {
+            parameterBuilder.hasDefaultValue = true
+        }
         return parameterBuilder.build()
     }
 
@@ -430,7 +455,12 @@ class JsAstSerializer {
             varDecl.initExpression?.let { declBuilder.initialValue = serialize(it) }
             varsBuilder.addDeclaration(declBuilder)
         }
-        varsBuilder.multiline = vars.isMultiline
+
+        if (vars.isMultiline) {
+            varsBuilder.multiline = true
+        }
+        vars.exportedPackage?.let { varsBuilder.exportedPackageId = serialize(it) }
+
         return varsBuilder.build()
     }
 
@@ -493,6 +523,12 @@ class JsAstSerializer {
         JsUnaryOperator.VOID -> VOID
     }
 
+    private fun map(sideEffects: SideEffectKind) = when (sideEffects) {
+        SideEffectKind.AFFECTS_STATE -> SideEffects.AFFECTS_STATE
+        SideEffectKind.DEPENDS_ON_STATE -> SideEffects.DEPENDS_ON_STATE
+        SideEffectKind.PURE -> SideEffects.PURE
+    }
+
     private fun serialize(name: JsName) = nameMap.getOrPut(name) {
         val result = nameTableBuilder.entryCount
         val builder = Name.newBuilder()
@@ -507,4 +543,51 @@ class JsAstSerializer {
         stringTableBuilder.addEntry(string)
         result
     }
+
+    private inline fun withLocation(node: JsNode, fileConsumer: (Int) -> Unit, locationConsumer: (Location) -> Unit, inner: () -> Unit) {
+        val lastLocation = locationStack.peek()
+        val location = extractLocation(node)
+        val locationStackModified = if (lastLocation != location && location != null) {
+            locationStack.push(location)
+            if (lastLocation == null || lastLocation.file != location.file) {
+                fileConsumer(serialize(location.file))
+            }
+            val locationBuilder = Location.newBuilder()
+            locationBuilder.startLine = location.startLine
+            locationBuilder.startChar = location.startChar
+            locationConsumer(locationBuilder.build())
+            true
+        }
+        else {
+            false
+        }
+
+        inner()
+
+        if (locationStackModified) {
+            locationStack.pop()
+        }
+    }
+
+    private fun extractLocation(node: JsNode): JsLocation? {
+        val source = node.source
+        return when (source) {
+            is JsLocation -> source
+            is PsiElement -> extractLocation(source)
+            else -> null
+        }
+    }
+
+    private fun extractLocation(element: PsiElement): JsLocation {
+        val file = element.containingFile
+        val document = file.viewProvider.document!!
+
+        val path = file.viewProvider.virtualFile.path
+
+        val startOffset = element.node.startOffset
+        val startLine = document.getLineNumber(startOffset)
+        val startChar = startOffset - document.getLineStartOffset(startLine)
+
+        return JsLocation(path, startLine, startChar)
+    }
 }