JS IR: custom JS AST serialization

This commit is contained in:
Anton Bannykh
2023-01-07 22:45:52 +01:00
committed by Space Team
parent 978f937fa9
commit 698be0fddd
6 changed files with 1077 additions and 208 deletions
@@ -6,17 +6,18 @@
package org.jetbrains.kotlin.ir.backend.js.ic
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragment
import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstSerializer
import org.jetbrains.kotlin.ir.backend.js.utils.serialization.serializeTo
import org.jetbrains.kotlin.konan.file.use
import java.io.BufferedOutputStream
import java.io.File
internal sealed class SourceFileCacheArtifact(val srcFile: KotlinSourceFile, val binaryAstFile: File) {
abstract fun commitMetadata()
fun commitBinaryAst(fragment: JsIrProgramFragment, serializer: JsIrAstSerializer) {
fun commitBinaryAst(fragment: JsIrProgramFragment) {
binaryAstFile.recreate()
BufferedOutputStream(binaryAstFile.outputStream()).use { bufferedOutStream ->
serializer.serialize(fragment, bufferedOutStream)
BufferedOutputStream(binaryAstFile.outputStream()).use {
fragment.serializeTo(it)
}
}
@@ -61,12 +62,10 @@ internal class IncrementalCacheArtifact(
moduleName: String,
rebuiltFileFragments: Map<KotlinSourceFile, JsIrProgramFragment>,
): ModuleArtifact {
val serializer = JsIrAstSerializer()
val fileArtifacts = srcCacheActions.map { srcFileAction ->
val rebuiltFileFragment = rebuiltFileFragments[srcFileAction.srcFile]
if (rebuiltFileFragment != null) {
srcFileAction.commitBinaryAst(rebuiltFileFragment, serializer)
srcFileAction.commitBinaryAst(rebuiltFileFragment)
}
srcFileAction.commitMetadata()
SrcFileArtifact(srcFileAction.srcFile.path, rebuiltFileFragment, srcFileAction.binaryAstFile)
@@ -8,19 +8,16 @@ package org.jetbrains.kotlin.ir.backend.js.ic
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrModule
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragment
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.safeModuleName
import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstDeserializer
import java.io.ByteArrayInputStream
import org.jetbrains.kotlin.ir.backend.js.utils.serialization.deserializeJsIrProgramFragment
import java.io.File
class SrcFileArtifact(val srcFilePath: String, private val fragment: JsIrProgramFragment?, private val astArtifact: File? = null) {
fun loadJsIrFragment(deserializer: JsIrAstDeserializer): JsIrProgramFragment? {
fun loadJsIrFragment(): JsIrProgramFragment? {
if (fragment != null) {
return fragment
}
return astArtifact?.ifExists { readBytes() }?.let {
ByteArrayInputStream(it).use { byteStream ->
deserializer.deserialize(byteStream)
}
deserializeJsIrProgramFragment(it)
}
}
@@ -38,8 +35,7 @@ class ModuleArtifact(
val moduleExternalName = externalModuleName ?: moduleSafeName
fun loadJsIrModule(): JsIrModule {
val deserializer = JsIrAstDeserializer()
val fragments = fileArtifacts.sortedBy { it.srcFilePath }.mapNotNull { it.loadJsIrFragment(deserializer) }
val fragments = fileArtifacts.sortedBy { it.srcFilePath }.mapNotNull { it.loadJsIrFragment() }
return JsIrModule(moduleSafeName, moduleExternalName, fragments)
}
}
@@ -0,0 +1,57 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.utils.serialization
// TODO: is it OK to just use UTF_8?
val SerializationCharset = Charsets.UTF_8
object StatementIds {
const val RETURN = 0
const val THROW = 1
const val BREAK = 2
const val CONTINUE = 3
const val DEBUGGER = 4
const val EXPRESSION = 5
const val VARS = 6
const val BLOCK = 7
const val COMPOSITE_BLOCK = 8
const val LABEL = 9
const val IF = 10
const val SWITCH = 11
const val WHILE = 12
const val DO_WHILE = 13
const val FOR = 14
const val FOR_IN = 15
const val TRY = 16
const val EMPTY = 17
const val SINGLE_LINE_COMMENT = 18
const val MULTI_LINE_COMMENT = 19
}
object ExpressionIds {
const val THIS_REF = 0
const val NULL = 1
const val TRUE_LITERAL = 2
const val FALSE_LITERAL = 3
const val STRING_LITERAL = 4
const val REG_EXP = 5
const val INT_LITERAL = 6
const val DOUBLE_LITERAL = 7
const val ARRAY_LITERAL = 8
const val OBJECT_LITERAL = 9
const val FUNCTION = 10
const val DOC_COMMENT = 11
const val BINARY_OPERATION = 12
const val PREFIX_OPERATION = 13
const val POSTFIX_OPERATION = 14
const val CONDITIONAL = 15
const val ARRAY_ACCESS = 16
const val NAME_REFERENCE = 17
const val SIMPLE_NAME_REFERENCE = 18
const val PROPERTY_REFERENCE = 19
const val INVOCATION = 20
const val NEW = 21
}
@@ -11,97 +11,454 @@ import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragmen
import org.jetbrains.kotlin.ir.backend.js.utils.emptyScope
import org.jetbrains.kotlin.js.backend.ast.*
import org.jetbrains.kotlin.js.backend.ast.JsImportedModule
import org.jetbrains.kotlin.protobuf.CodedInputStream
import org.jetbrains.kotlin.serialization.js.ast.JsAstDeserializerBase
import org.jetbrains.kotlin.serialization.js.ast.JsAstProtoBuf.*
import java.io.InputStream
import org.jetbrains.kotlin.js.backend.ast.metadata.*
import org.jetbrains.kotlin.js.backend.ast.metadata.LocalAlias
import org.jetbrains.kotlin.js.backend.ast.metadata.SpecialFunction
import java.util.*
import java.util.ArrayDeque
class JsIrAstDeserializer : JsAstDeserializerBase() {
override val scope = emptyScope
fun deserialize(input: InputStream): JsIrProgramFragment {
return deserialize(Chunk.parseFrom(CodedInputStream.newInstance(input).apply { setRecursionLimit(4096) }))
}
fun deserialize(proto: Chunk): JsIrProgramFragment {
stringTable += proto.stringTable.entryList
nameTable += proto.nameTable.entryList
nameCache += nameTable.map { null }
try {
return deserialize(proto.fragment)
} finally {
stringTable.clear()
nameTable.clear()
nameCache.clear()
}
}
private fun deserialize(proto: Fragment): JsIrProgramFragment {
val fragment = JsIrProgramFragment(proto.packageFqn)
fragment.importedModules += proto.importedModuleList.map { importedModuleProto ->
JsImportedModule(
deserializeString(importedModuleProto.externalNameId),
deserializeName(importedModuleProto.internalNameId),
if (importedModuleProto.hasPlainReference()) deserialize(importedModuleProto.plainReference) else null
)
}
proto.importEntryList.associateTo(fragment.imports) { importProto ->
deserializeString(importProto.signatureId) to deserialize(importProto.expression)
}
if (proto.hasDeclarationBlock()) {
fragment.declarations.statements += deserializeCompositeBlock(proto.declarationBlock).statements
}
if (proto.hasInitializerBlock()) {
fragment.initializers.statements += deserializeCompositeBlock(proto.initializerBlock).statements
}
if (proto.hasExportBlock()) {
fragment.exports.statements += deserializeCompositeBlock(proto.exportBlock).statements
}
if (proto.hasPolyfills()) {
fragment.polyfills.statements += deserializeCompositeBlock(proto.polyfills).statements
}
proto.nameBindingList.associateTo(fragment.nameBindings) { nameBindingProto ->
deserializeString(nameBindingProto.signatureId) to deserializeName(nameBindingProto.nameId)
}
proto.optionalCrossModuleImportsList.mapTo(fragment.optionalCrossModuleImports) { deserializeString(it) }
proto.irClassModelList.associateTo(fragment.classes) { clsProto -> deserialize(clsProto) }
if (proto.hasTestsInvocation()) {
fragment.testFunInvocation = deserialize(proto.testsInvocation)
}
if (proto.hasMainInvocation()) {
fragment.mainFunction = deserialize(proto.mainInvocation)
}
if (proto.hasSuiteFunction()) {
fragment.suiteFn = deserializeName(proto.suiteFunction)
}
if (proto.hasDts()) {
fragment.dts = TypeScriptFragment(proto.dts)
}
fragment.definitions += proto.definitionsList.map { deserializeString(it) }
return fragment
}
private fun deserialize(proto: IrClassModel): Pair<JsName, JsIrIcClassModel> {
return deserializeName(proto.nameId) to JsIrIcClassModel(proto.superClassesList.map { deserializeName(it) }).apply {
if (proto.hasPreDeclarationBlock()) {
preDeclarationBlock.statements += deserializeCompositeBlock(proto.preDeclarationBlock).statements
}
if (proto.hasPostDeclarationBlock()) {
postDeclarationBlock.statements += deserializeCompositeBlock(proto.postDeclarationBlock).statements
}
}
}
override fun embedSources(deserializedLocation: JsLocation, file: String): JsLocationWithSource? = null
fun deserializeJsIrProgramFragment(input: ByteArray): JsIrProgramFragment {
return JsIrAstDeserializer(input).readFragment()
}
private class JsIrAstDeserializer(private val source: ByteArray) {
private var offset = 0
private val stringTable = readArray { readString() }
private val nameTable = readArray { readName() }
private val scope = emptyScope
private val fileStack: Deque<String> = ArrayDeque()
private fun readByte(): Byte {
return source[offset++]
}
private fun readBoolean(): Boolean {
return readByte() != 0.toByte()
}
private fun readInt(): Int {
var result = readByte().toInt() // no need to mask the highest bit
result = (result shl 8) or (readByte().toInt() and 0xFF)
result = (result shl 8) or (readByte().toInt() and 0xFF)
result = (result shl 8) or (readByte().toInt() and 0xFF)
return result
}
private fun readLong(): Long {
var result = readByte().toLong() // no need to mask the highest bit
result = (result shl 8) or (readByte().toLong() and 0xFF)
result = (result shl 8) or (readByte().toLong() and 0xFF)
result = (result shl 8) or (readByte().toLong() and 0xFF)
result = (result shl 8) or (readByte().toLong() and 0xFF)
result = (result shl 8) or (readByte().toLong() and 0xFF)
result = (result shl 8) or (readByte().toLong() and 0xFF)
result = (result shl 8) or (readByte().toLong() and 0xFF)
return result
}
private fun readDouble(): Double {
return Double.fromBits(readLong())
}
private fun readString(): String {
val length = readInt()
val result = String(source, offset, length, SerializationCharset)
offset += length
return result
}
private inline fun <reified T> readArray(readElement: () -> T): Array<T> {
return Array<T>(readInt()) { readElement() }
}
private inline fun readRepeated(readElement: () -> Unit) {
var length = readInt()
while (length-- > 0) {
readElement()
}
}
private inline fun <T> readList(readElement: () -> T): List<T> {
val length = readInt()
val result = ArrayList<T>(length)
for (i in 0 until length) {
result.add(readElement())
}
return result
}
private inline fun <T> ifTrue(then: () -> T): T? {
return if (readBoolean()) then() else null
}
fun readFragment(): JsIrProgramFragment {
return JsIrProgramFragment(readString()).apply {
readRepeated {
importedModules += JsImportedModule(
externalName = stringTable[readInt()],
internalName = nameTable[readInt()],
plainReference = ifTrue { readExpression() }
)
}
readRepeated { imports[stringTable[readInt()]] = readExpression() }
readRepeated { declarations.statements += readStatement() }
readRepeated { initializers.statements += readStatement() }
readRepeated { exports.statements += readStatement() }
readRepeated { polyfills.statements += readStatement() }
readRepeated { nameBindings[stringTable[readInt()]] = nameTable[readInt()] }
readRepeated { optionalCrossModuleImports.add(stringTable[readInt()]) }
readRepeated { classes[nameTable[readInt()]] = readIrIcClassModel() }
ifTrue { testFunInvocation = readStatement() }
ifTrue { mainFunction = readStatement() }
ifTrue { dts = TypeScriptFragment(readString()) }
ifTrue { suiteFn = nameTable[readInt()] }
readRepeated { definitions += stringTable[readInt()] }
}
}
private fun readIrIcClassModel(): JsIrIcClassModel {
return JsIrIcClassModel(readList { nameTable[readInt()] }).apply {
readRepeated { preDeclarationBlock.statements += readStatement() }
readRepeated { postDeclarationBlock.statements += readStatement() }
}
}
private fun readStatement(): JsStatement {
return withComments {
withLocation {
with(StatementIds) {
when (val id = readByte().toInt()) {
RETURN -> {
JsReturn(ifTrue { readExpression() })
}
THROW -> {
JsThrow(readExpression())
}
BREAK -> {
JsBreak(ifTrue { JsNameRef(nameTable[readInt()]) })
}
CONTINUE -> {
JsContinue(ifTrue { JsNameRef(nameTable[readInt()]) })
}
DEBUGGER -> {
JsDebugger()
}
EXPRESSION -> {
JsExpressionStatement(readExpression()).apply {
ifTrue { exportedTag = stringTable[readInt()] }
}
}
VARS -> {
readVars()
}
BLOCK -> {
JsBlock().apply {
readRepeated { statements += readStatement() }
}
}
COMPOSITE_BLOCK -> {
readCompositeBlock()
}
LABEL -> {
JsLabel(nameTable[readInt()], readStatement())
}
IF -> {
JsIf(readExpression(), readStatement(), ifTrue { readStatement() })
}
SWITCH -> {
JsSwitch(
readExpression(),
readList {
withLocation {
ifTrue {
JsCase().apply { caseExpression = readExpression() }
} ?: JsDefault()
}.apply {
readRepeated {
statements += readStatement()
}
}
}
)
}
WHILE -> {
JsWhile(readExpression(), readStatement())
}
DO_WHILE -> {
JsDoWhile(readExpression(), readStatement())
}
FOR -> {
val condition = ifTrue { readExpression() }
val incrementExpression = ifTrue { readExpression() }
val body = ifTrue { readStatement() }
ifTrue {
JsFor(
readVars(),
condition,
incrementExpression,
body
)
} ?: JsFor(
ifTrue { readExpression() },
condition,
incrementExpression,
body
)
}
FOR_IN -> {
JsForIn(
ifTrue { nameTable[readInt()] },
ifTrue { readExpression() },
readExpression(),
readStatement()
)
}
TRY -> {
JsTry(
readBlock(),
readList {
JsCatch(nameTable[readInt()]).apply {
body = readBlock()
}
},
ifTrue { readBlock() }
)
}
EMPTY -> {
JsEmpty
}
SINGLE_LINE_COMMENT -> {
JsSingleLineComment(readString())
}
MULTI_LINE_COMMENT -> {
JsMultiLineComment(readString())
}
else -> error("Unknown statement id: $id")
}
}
}
}.apply {
synthetic = readBoolean()
}
}
private val sideEffectKindValues = SideEffectKind.values()
private val jsBinaryOperatorValues = JsBinaryOperator.values()
private val jsUnaryOperatorValues = JsUnaryOperator.values()
private fun readExpression(): JsExpression {
return withComments {
withLocation {
with(ExpressionIds) {
when (val id = readByte().toInt()) {
THIS_REF -> {
JsThisRef()
}
NULL -> {
JsNullLiteral()
}
TRUE_LITERAL -> {
JsBooleanLiteral(true)
}
FALSE_LITERAL -> {
JsBooleanLiteral(false)
}
STRING_LITERAL -> {
JsStringLiteral(stringTable[readInt()])
}
REG_EXP -> {
JsRegExp().apply {
pattern = stringTable[readInt()]
ifTrue { flags = stringTable[readInt()] }
}
}
INT_LITERAL -> {
JsIntLiteral(readInt())
}
DOUBLE_LITERAL -> {
JsDoubleLiteral(readDouble())
}
ARRAY_LITERAL -> {
JsArrayLiteral(readList { readExpression() })
}
OBJECT_LITERAL -> {
JsObjectLiteral(
readList { JsPropertyInitializer(readExpression(), readExpression()) },
readBoolean()
)
}
FUNCTION -> {
JsFunction(scope, readBlock(), "").apply {
readRepeated { parameters += readParameter() }
ifTrue { name = nameTable[readInt()] }
isLocal = readBoolean()
}
}
DOC_COMMENT -> {
val tags = hashMapOf<String, Any>()
readRepeated {
tags[stringTable[readInt()]] = ifTrue { readExpression() } ?: stringTable[readInt()]
}
JsDocComment(tags)
}
BINARY_OPERATION -> {
JsBinaryOperation(
jsBinaryOperatorValues[readInt()],
readExpression(),
readExpression()
)
}
PREFIX_OPERATION -> {
JsPrefixOperation(jsUnaryOperatorValues[readInt()], readExpression())
}
POSTFIX_OPERATION -> {
JsPostfixOperation(jsUnaryOperatorValues[readInt()], readExpression())
}
CONDITIONAL -> {
JsConditional(
readExpression(),
readExpression(),
readExpression()
)
}
ARRAY_ACCESS -> {
JsArrayAccess(readExpression(), readExpression())
}
NAME_REFERENCE -> {
JsNameRef(nameTable[readInt()]).apply {
ifTrue { qualifier = readExpression() }
ifTrue { isInline = readBoolean() }
}
}
SIMPLE_NAME_REFERENCE -> {
JsNameRef(nameTable[readInt()])
}
PROPERTY_REFERENCE -> {
JsNameRef(stringTable[readInt()]).apply {
ifTrue { qualifier = readExpression() }
ifTrue { isInline = readBoolean() }
}
}
INVOCATION -> {
JsInvocation(readExpression(), readList { readExpression() }).apply {
ifTrue { isInline = readBoolean() }
}
}
NEW -> {
JsNew(readExpression(), readList { readExpression() })
}
else -> error("Unknown expression id: $id")
}
}
}
}.apply {
synthetic = readBoolean()
sideEffects = sideEffectKindValues[readInt()]
ifTrue { localAlias = readJsImportedModule() }
}
}
private fun readJsImportedModule(): JsImportedModule {
return JsImportedModule(
stringTable[readInt()],
nameTable[readInt()],
ifTrue { readExpression() }
)
}
private fun readParameter(): JsParameter {
return JsParameter(nameTable[readInt()]).apply {
hasDefaultValue = readBoolean()
}
}
private fun readCompositeBlock(): JsCompositeBlock {
return JsCompositeBlock().apply {
readRepeated { statements += readStatement() }
}
}
private fun readBlock(): JsBlock {
return ifTrue { readCompositeBlock() } ?: JsBlock().apply {
readRepeated { statements += readStatement() }
}
}
private fun readVars(): JsVars {
return JsVars(readBoolean()).apply {
readRepeated {
vars += withLocation {
JsVars.JsVar(nameTable[readInt()], ifTrue { readExpression() })
}
}
ifTrue { exportedPackage = stringTable[readInt()] }
}
}
private val specialFunctionValues = SpecialFunction.values()
private fun readName(): JsName {
val identifier = stringTable[readInt()]
val name = ifTrue {
JsScope.declareTemporaryName(identifier)
} ?: JsDynamicScope.declareName(identifier)
ifTrue { name.localAlias = readLocalAlias() }
ifTrue { name.imported = true }
ifTrue { name.specialFunction = specialFunctionValues[readInt()] }
return name
}
private fun readLocalAlias(): LocalAlias {
return LocalAlias(
nameTable[readInt()],
ifTrue { stringTable[readInt()] }
)
}
private fun readComment(): JsComment {
val text = readString()
return ifTrue { JsMultiLineComment(text) } ?: JsSingleLineComment(text)
}
private inline fun <T : JsNode> withLocation(action: () -> T): T {
return ifTrue {
val deserializedFile = ifTrue { stringTable[readInt()] }
val file = deserializedFile ?: fileStack.peek()
val startLine = readInt()
val startChar = readInt()
val deserializedLocation = file?.let { JsLocation(it, startLine, startChar) }
val shouldUpdateFile = deserializedFile != null && deserializedFile != fileStack.peek()
if (shouldUpdateFile) {
fileStack.push(deserializedFile)
}
val node = action()
if (deserializedLocation != null) {
node.source = deserializedLocation
}
if (shouldUpdateFile) {
fileStack.pop()
}
node
} ?: action()
}
private inline fun <T : JsNode> withComments(action: () -> T): T {
return action().apply {
ifTrue { this.commentsBeforeNode = readArray { readComment() }.toList() }
ifTrue { this.commentsAfterNode = readArray { readComment() }.toList() }
}
}
}
@@ -5,114 +5,577 @@
package org.jetbrains.kotlin.ir.backend.js.utils.serialization
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrClassModel
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrIcClassModel
import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragment
import org.jetbrains.kotlin.js.backend.ast.*
import org.jetbrains.kotlin.serialization.js.ast.JsAstProtoBuf.*
import org.jetbrains.kotlin.serialization.js.ast.JsAstSerializerBase
import org.jetbrains.kotlin.utils.addToStdlib.ifNotEmpty
import org.jetbrains.kotlin.js.backend.ast.JsImportedModule
import org.jetbrains.kotlin.js.backend.ast.metadata.*
import org.jetbrains.kotlin.utils.addToStdlib.safeAs
import java.io.ByteArrayOutputStream
import java.io.DataOutputStream
import java.io.OutputStream
import java.util.*
import java.util.ArrayDeque
class JsIrAstSerializer: JsAstSerializerBase() {
fun JsIrProgramFragment.serializeTo(output: OutputStream) {
JsIrAstSerializer().append(this).saveTo(output)
}
fun serialize(fragment: JsIrProgramFragment, output: OutputStream) {
importedNames.clear()
importedNames += fragment.imports.map { fragment.nameBindings[it.key]!! }
serialize(fragment).writeTo(output)
}
private class DataWriter {
fun serialize(fragment: JsIrProgramFragment): Chunk {
try {
val chunkBuilder = Chunk.newBuilder()
chunkBuilder.fragment = serializeFragment(fragment)
chunkBuilder.nameTable = nameTableBuilder.build()
chunkBuilder.stringTable = stringTableBuilder.build()
return chunkBuilder.build()
} finally {
nameTableBuilder.clear()
stringTableBuilder.clear()
nameMap.clear()
stringMap.clear()
val data = ByteArrayOutputStream()
private val output = DataOutputStream(data)
fun saveTo(output: DataOutputStream) {
output.let {
data.writeTo(it)
}
}
private fun serializeFragment(fragment: JsIrProgramFragment): Fragment {
val fragmentBuilder = Fragment.newBuilder()
fragmentBuilder.packageFqn = fragment.packageFqn
for (importedModule in fragment.importedModules) {
val importedModuleBuilder = ImportedModule.newBuilder()
importedModuleBuilder.externalNameId = serialize(importedModule.externalName)
importedModuleBuilder.internalNameId = serialize(importedModule.internalName)
importedModule.plainReference?.let { importedModuleBuilder.plainReference = serialize(it) }
fragmentBuilder.addImportedModule(importedModuleBuilder)
}
for ((signature, expression) in fragment.imports) {
val importBuilder = Import.newBuilder()
importBuilder.signatureId = serialize(signature)
importBuilder.expression = serialize(expression)
fragmentBuilder.addImportEntry(importBuilder)
}
fragmentBuilder.declarationBlock = serializeBlock(fragment.declarations)
fragmentBuilder.initializerBlock = serializeBlock(fragment.initializers)
fragmentBuilder.exportBlock = serializeBlock(fragment.exports)
fragmentBuilder.polyfills = serializeBlock(fragment.polyfills)
for ((key, name) in fragment.nameBindings.entries) {
val nameBindingBuilder = NameBinding.newBuilder()
nameBindingBuilder.signatureId = serialize(key)
nameBindingBuilder.nameId = serialize(name)
fragmentBuilder.addNameBinding(nameBindingBuilder)
}
fragment.optionalCrossModuleImports.forEach {
fragmentBuilder.addOptionalCrossModuleImports(serialize(it))
}
fragment.classes.entries.forEach { (name, model) -> fragmentBuilder.addIrClassModel(serialize(name, model)) }
fragment.testFunInvocation?.let {
fragmentBuilder.setTestsInvocation(serialize(it))
}
fragment.mainFunction?.let {
fragmentBuilder.setMainInvocation(serialize(it))
}
fragment.dts?.let {
fragmentBuilder.dts = it.raw
}
fragment.suiteFn?.let {
fragmentBuilder.setSuiteFunction(serialize(it))
}
fragment.definitions.forEach {
fragmentBuilder.addDefinitions(serialize(it))
}
return fragmentBuilder.build()
fun writeByte(byte: Int) {
// Limit bytes to positive values to avoid conversion in deserializer
if ((byte and 0x7F.inv()) != 0) error("Byte out of bounds: $byte")
output.writeByte(byte)
}
private fun serialize(name: JsName, classModel: JsIrIcClassModel): IrClassModel {
val builder = IrClassModel.newBuilder()
builder.nameId = serialize(name)
classModel.superClasses.forEach { builder.addSuperClasses(serialize(it)) }
if (classModel.preDeclarationBlock.statements.isNotEmpty()) {
builder.preDeclarationBlock = serializeBlock(classModel.preDeclarationBlock)
}
if (classModel.postDeclarationBlock.statements.isNotEmpty()) {
builder.postDeclarationBlock = serializeBlock(classModel.postDeclarationBlock)
}
return builder.build()
fun writeByteArray(byteArray: ByteArray) {
output.writeInt(byteArray.size)
output.write(byteArray)
}
override fun extractLocation(node: JsNode): JsLocation? {
return node.source.safeAs<JsLocationWithSource>()?.asSimpleLocation()
fun writeString(string: String) {
writeByteArray(string.toByteArray(SerializationCharset))
}
fun writeBoolean(boolean: Boolean) {
output.writeBoolean(boolean)
}
fun writeInt(int: Int) {
output.writeInt(int)
}
fun writeDouble(double: Double) {
output.writeDouble(double)
}
inline fun <T> writeCollection(collection: Collection<T>, writeItem: (T) -> Unit) {
output.writeInt(collection.size)
collection.forEach(writeItem)
}
inline fun <T> ifNotNull(t: T?, write: (T) -> Unit): T? {
output.writeBoolean(t != null)
if (t != null) {
write(t)
}
return t
}
inline fun ifTrue(condition: Boolean, write: () -> Unit) {
output.writeBoolean(condition)
if (condition) {
write()
}
}
}
private class JsIrAstSerializer {
private val fragmentSerializer = DataWriter()
private val nameSerializer = DataWriter()
private val stringSerializer = DataWriter()
private val nameMap = mutableMapOf<JsName, Int>()
private val stringMap = mutableMapOf<String, Int>()
private val fileStack: Deque<String> = ArrayDeque()
private val importedNames = mutableSetOf<JsName>()
fun append(fragment: JsIrProgramFragment): JsIrAstSerializer {
importedNames += fragment.imports.map { fragment.nameBindings[it.key]!! }
fragmentSerializer.writeFragment(fragment)
return this
}
fun saveTo(rawOutput: OutputStream) {
DataOutputStream(rawOutput).use {
it.writeInt(stringMap.size)
stringSerializer.saveTo(it)
it.writeInt(nameMap.size)
nameSerializer.saveTo(it)
fragmentSerializer.saveTo(it)
}
}
private fun DataWriter.writeFragment(fragment: JsIrProgramFragment) {
writeString(fragment.packageFqn)
writeCollection(fragment.importedModules) {
writeInt(internalizeString(it.externalName))
writeInt(internalizeName(it.internalName))
ifNotNull(it.plainReference) {
writeExpression(it)
}
}
writeCollection(fragment.imports.entries) { (signatureId, expression) ->
writeInt(internalizeString(signatureId))
writeExpression(expression)
}
writeCompositeBlock(fragment.declarations)
writeCompositeBlock(fragment.initializers)
writeCompositeBlock(fragment.exports)
writeCompositeBlock(fragment.polyfills)
writeCollection(fragment.nameBindings.entries) { (key, name) ->
writeInt(internalizeString(key))
writeInt(internalizeName(name))
}
writeCollection(fragment.optionalCrossModuleImports) {
writeInt(internalizeString(it))
}
writeCollection(fragment.classes.entries) { (name, model) ->
writeInt(internalizeName(name))
writeIrIcModel(model)
}
ifNotNull(fragment.testFunInvocation) {
writeStatement(it)
}
ifNotNull(fragment.mainFunction) {
writeStatement(it)
}
ifNotNull(fragment.dts) {
writeString(it.raw)
}
ifNotNull(fragment.suiteFn) {
writeInt(internalizeName(it))
}
writeCollection(fragment.definitions) {
writeInt(internalizeString(it))
}
}
private fun DataWriter.writeIrIcModel(classModel: JsIrIcClassModel) {
writeCollection(classModel.superClasses) {
writeInt(internalizeName(it))
}
writeCompositeBlock(classModel.preDeclarationBlock)
writeCompositeBlock(classModel.postDeclarationBlock)
}
private fun DataWriter.writeStatement(statement: JsStatement) {
val visitor = object : JsVisitor() {
override fun visitReturn(x: JsReturn) {
writeByte(StatementIds.RETURN)
ifNotNull(x.expression) {
writeExpression(it)
}
}
override fun visitThrow(x: JsThrow) {
writeByte(StatementIds.THROW)
writeExpression(x.expression)
}
override fun visitBreak(x: JsBreak) {
writeByte(StatementIds.BREAK)
ifNotNull(x.label) {
writeInt(internalizeName(it.name!!))
}
}
override fun visitContinue(x: JsContinue) {
writeByte(StatementIds.CONTINUE)
ifNotNull(x.label) {
writeInt(internalizeName(it.name!!))
}
}
override fun visitDebugger(x: JsDebugger) {
writeByte(StatementIds.DEBUGGER)
}
override fun visitExpressionStatement(x: JsExpressionStatement) {
writeByte(StatementIds.EXPRESSION)
writeExpression(x.expression)
ifNotNull(x.exportedTag) { writeInt(internalizeString(it)) }
}
override fun visitVars(x: JsVars) {
writeByte(StatementIds.VARS)
writeVars(x)
}
override fun visitBlock(x: JsBlock) {
if (x is JsCompositeBlock) {
writeByte(StatementIds.COMPOSITE_BLOCK)
writeCompositeBlock(x)
} else {
writeByte(StatementIds.BLOCK)
writeCollection(x.statements) {
writeStatement(it)
}
}
}
override fun visitLabel(x: JsLabel) {
writeByte(StatementIds.LABEL)
writeInt(internalizeName(x.name))
writeStatement(x.statement)
}
override fun visitIf(x: JsIf) {
writeByte(StatementIds.IF)
writeExpression(x.ifExpression)
writeStatement(x.thenStatement)
ifNotNull(x.elseStatement) { writeStatement(it) }
}
override fun visit(x: JsSwitch) {
writeByte(StatementIds.SWITCH)
writeExpression(x.expression)
writeCollection(x.cases) { case ->
withLocation(case) {
ifNotNull(case as? JsCase) {
writeExpression(it.caseExpression)
}
}
writeCollection(case.statements) { part ->
writeStatement(part)
}
}
}
override fun visitWhile(x: JsWhile) {
writeByte(StatementIds.WHILE)
writeExpression(x.condition)
writeStatement(x.body)
}
override fun visitDoWhile(x: JsDoWhile) {
writeByte(StatementIds.DO_WHILE)
writeExpression(x.condition)
writeStatement(x.body)
}
override fun visitFor(x: JsFor) {
writeByte(StatementIds.FOR)
ifNotNull(x.condition) { writeExpression(it) }
ifNotNull(x.incrementExpression) { writeExpression(it) }
ifNotNull(x.body) { writeStatement(it) }
ifNotNull(x.initVars) {
writeVars(it)
} ?: ifNotNull(x.initExpression) {
writeExpression(it)
}
}
override fun visitForIn(x: JsForIn) {
writeByte(StatementIds.FOR_IN)
ifNotNull(x.iterVarName) {
writeInt(internalizeName(it))
}
ifNotNull(x.iterExpression) { writeExpression(it) }
writeExpression(x.objectExpression)
writeStatement(x.body)
}
override fun visitTry(x: JsTry) {
writeByte(StatementIds.TRY)
writeBlock(x.tryBlock)
writeCollection(x.catches) { c ->
writeInt(internalizeName(c.parameter.name))
writeBlock(c.body)
}
ifNotNull(x.finallyBlock) { writeBlock(it) }
}
override fun visitEmpty(x: JsEmpty) {
writeByte(StatementIds.EMPTY)
}
override fun visitSingleLineComment(comment: JsSingleLineComment) {
writeByte(StatementIds.SINGLE_LINE_COMMENT)
writeString(comment.text)
}
override fun visitMultiLineComment(comment: JsMultiLineComment) {
writeByte(StatementIds.MULTI_LINE_COMMENT)
writeString(comment.text)
}
override fun visitElement(node: JsNode) {
error("Unknown statement type: ${statement::class.qualifiedName}")
}
}
withComments(statement) {
withLocation(statement) {
statement.accept(visitor)
}
}
writeBoolean((statement as HasMetadata).synthetic)
}
private fun DataWriter.writeExpression(expression: JsExpression) {
val visitor = object : JsVisitor() {
override fun visitThis(x: JsThisRef) {
writeByte(ExpressionIds.THIS_REF)
}
override fun visitNull(x: JsNullLiteral) {
writeByte(ExpressionIds.NULL)
}
override fun visitBoolean(x: JsBooleanLiteral) {
if (x.value) {
writeByte(ExpressionIds.TRUE_LITERAL)
} else {
writeByte(ExpressionIds.FALSE_LITERAL)
}
}
override fun visitString(x: JsStringLiteral) {
writeByte(ExpressionIds.STRING_LITERAL)
writeInt(internalizeString(x.value))
}
override fun visitRegExp(x: JsRegExp) {
writeByte(ExpressionIds.REG_EXP)
writeInt(internalizeString(x.pattern))
ifNotNull(x.flags) { writeInt(internalizeString(it)) }
}
override fun visitInt(x: JsIntLiteral) {
writeByte(ExpressionIds.INT_LITERAL)
writeInt(x.value)
}
override fun visitDouble(x: JsDoubleLiteral) {
writeByte(ExpressionIds.DOUBLE_LITERAL)
writeDouble(x.value)
}
override fun visitArray(x: JsArrayLiteral) {
writeByte(ExpressionIds.ARRAY_LITERAL)
writeCollection(x.expressions) {
writeExpression(it)
}
}
override fun visitObjectLiteral(x: JsObjectLiteral) {
writeByte(ExpressionIds.OBJECT_LITERAL)
writeCollection(x.propertyInitializers) {
writeExpression(it.labelExpr)
writeExpression(it.valueExpr)
}
writeBoolean(x.isMultiline)
}
override fun visitFunction(x: JsFunction) {
writeByte(ExpressionIds.FUNCTION)
writeBlock(x.body)
writeCollection(x.parameters) { writeParameter(it) }
ifNotNull(x.name) {
writeInt(internalizeName(it))
}
writeBoolean(x.isLocal)
}
override fun visitDocComment(comment: JsDocComment) {
writeByte(ExpressionIds.DOC_COMMENT)
writeCollection(comment.tags.entries) { (name, value) ->
writeInt(internalizeString(name))
ifNotNull(value as? JsNameRef) {
writeExpression(it)
} ?: if (value is String) {
writeInt(internalizeString(value))
} else {
error("Unsupported tag: ${value::class.qualifiedName}")
}
}
}
override fun visitBinaryExpression(x: JsBinaryOperation) {
writeByte(ExpressionIds.BINARY_OPERATION)
writeInt(x.operator.ordinal)
writeExpression(x.arg1)
writeExpression(x.arg2)
}
override fun visitPrefixOperation(x: JsPrefixOperation) {
writeByte(ExpressionIds.PREFIX_OPERATION)
writeInt(x.operator.ordinal)
writeExpression(x.arg)
}
override fun visitPostfixOperation(x: JsPostfixOperation) {
writeByte(ExpressionIds.POSTFIX_OPERATION)
writeInt(x.operator.ordinal)
writeExpression(x.arg)
}
override fun visitConditional(x: JsConditional) {
writeByte(ExpressionIds.CONDITIONAL)
writeExpression(x.testExpression)
writeExpression(x.thenExpression)
writeExpression(x.elseExpression)
}
override fun visitArrayAccess(x: JsArrayAccess) {
writeByte(ExpressionIds.ARRAY_ACCESS)
writeExpression(x.arrayExpression)
writeExpression(x.indexExpression)
}
override fun visitNameRef(nameRef: JsNameRef) {
val name = nameRef.name
val qualifier = nameRef.qualifier
if (name != null) {
if (qualifier != null || nameRef.isInline == true) {
writeByte(ExpressionIds.NAME_REFERENCE)
writeInt(internalizeName(name))
ifNotNull(qualifier) { writeExpression(it) }
ifNotNull(nameRef.isInline) { writeBoolean(it) }
} else {
writeByte(ExpressionIds.SIMPLE_NAME_REFERENCE)
writeInt(internalizeName(name))
}
} else {
writeByte(ExpressionIds.PROPERTY_REFERENCE)
writeInt(internalizeString(nameRef.ident))
ifNotNull(qualifier) { writeExpression(it) }
ifNotNull(nameRef.isInline) { writeBoolean(it) }
}
}
override fun visitInvocation(invocation: JsInvocation) {
writeByte(ExpressionIds.INVOCATION)
writeExpression(invocation.qualifier)
writeCollection(invocation.arguments) { writeExpression(it) }
ifNotNull(invocation.isInline) { writeBoolean(it) }
}
override fun visitNew(x: JsNew) {
writeByte(ExpressionIds.NEW)
writeExpression(x.constructorExpression)
writeCollection(x.arguments) { writeExpression(it) }
}
}
withComments(expression) {
withLocation(expression) {
expression.accept(visitor)
}
}
writeBoolean(expression.synthetic)
writeInt(expression.sideEffects.ordinal)
ifNotNull(expression.localAlias) { writeImportedModule(it) }
}
private fun DataWriter.writeImportedModule(module: JsImportedModule) {
writeInt(internalizeString(module.externalName))
writeInt(internalizeName(module.internalName))
ifNotNull(module.plainReference) { writeExpression(it) }
}
private fun DataWriter.writeParameter(parameter: JsParameter) {
writeInt(internalizeName(parameter.name))
writeBoolean(parameter.hasDefaultValue)
}
private fun DataWriter.writeCompositeBlock(block: JsCompositeBlock) {
writeCollection(block.statements) { writeStatement(it) }
}
private fun DataWriter.writeBlock(block: JsBlock) {
writeBoolean(block is JsCompositeBlock)
writeCollection(block.statements) { writeStatement(it) }
}
private fun DataWriter.writeVars(vars: JsVars) {
writeBoolean(vars.isMultiline)
writeCollection(vars.vars) { varDecl ->
withLocation(varDecl) {
writeInt(internalizeName(varDecl.name))
ifNotNull(varDecl.initExpression) { writeExpression(it) }
}
}
ifNotNull(vars.exportedPackage) { writeInt(internalizeString(it)) }
}
private fun internalizeName(name: JsName): Int = nameMap.getOrPut(name) {
// Make sure all dependant JsName's are already serialized.
name.localAlias?.let { internalizeName(it.name) }
nameSerializer.apply {
writeInt(internalizeString(name.ident))
writeBoolean(name.isTemporary)
ifNotNull(name.localAlias) { writeLocalAlias(it) }
writeBoolean(name.imported && name !in importedNames)
ifNotNull(name.specialFunction) { writeInt(it.ordinal) }
}
nameMap.size
}
private fun DataWriter.writeLocalAlias(alias: LocalAlias) {
writeInt(internalizeName(alias.name))
ifNotNull(alias.tag) { writeInt(internalizeString(it)) }
}
private fun internalizeString(string: String) = stringMap.getOrPut(string) {
stringSerializer.writeString(string)
stringMap.size
}
private fun DataWriter.writeComment(comment: JsComment) {
writeString(comment.text)
when (comment) {
is JsSingleLineComment -> writeBoolean(false)
is JsMultiLineComment -> writeBoolean(true)
else -> error("Unknown type of comment ${comment.javaClass.name}")
}
}
private inline fun DataWriter.withLocation(node: JsNode, inner: () -> Unit) {
val location = node.source.safeAs<JsLocationWithSource>()?.asSimpleLocation()
ifNotNull(location) {
val lastFile = fileStack.peek()
val newFile = it.file
val fileChanged = lastFile != newFile
ifTrue(fileChanged) {
writeInt(internalizeString(newFile))
fileStack.push(it.file)
}
writeInt(it.startLine)
writeInt(it.startChar)
inner()
if (fileChanged) {
fileStack.pop()
}
} ?: inner()
}
private inline fun DataWriter.withComments(node: JsNode, inner: () -> Unit) {
inner()
ifNotNull(node.commentsBeforeNode) { comments -> writeCollection(comments) { writeComment(it) } }
ifNotNull(node.commentsAfterNode) { comments -> writeCollection(comments) { writeComment(it) } }
}
}
@@ -9,8 +9,8 @@ import org.jetbrains.kotlin.config.CompilerConfiguration
import org.jetbrains.kotlin.ir.backend.js.WholeWorldStageController
import org.jetbrains.kotlin.ir.backend.js.ic.*
import org.jetbrains.kotlin.ir.backend.js.moduleName
import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstDeserializer
import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstSerializer
import org.jetbrains.kotlin.ir.backend.js.utils.serialization.serializeTo
import org.jetbrains.kotlin.ir.backend.js.utils.serialization.deserializeJsIrProgramFragment
import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC
import org.jetbrains.kotlin.js.test.handlers.JsBoxRunner
import org.jetbrains.kotlin.konan.properties.propertyList
@@ -24,13 +24,11 @@ import org.jetbrains.kotlin.test.services.TestServices
import org.jetbrains.kotlin.test.services.compilerConfigurationProvider
import org.jetbrains.kotlin.test.services.configuration.JsEnvironmentConfigurator
import org.jetbrains.kotlin.test.services.jsLibraryProvider
import java.io.ByteArrayInputStream
import java.io.ByteArrayOutputStream
import java.io.File
private class TestArtifactCache(val moduleName: String, val binaryAsts: MutableMap<String, ByteArray> = mutableMapOf()) {
fun fetchArtifacts(): ModuleArtifact {
val deserializer = JsIrAstDeserializer()
return ModuleArtifact(
moduleName = moduleName,
fileArtifacts = binaryAsts.entries.map {
@@ -39,7 +37,7 @@ private class TestArtifactCache(val moduleName: String, val binaryAsts: MutableM
// TODO: It will be better to use saved fragments, but it doesn't work
// Merger.merge() + JsNode.resolveTemporaryNames() modify fragments,
// therefore the sequential calls produce different results
fragment = deserializer.deserialize(ByteArrayInputStream(it.value))
fragment = deserializeJsIrProgramFragment(it.value)
)
}
)
@@ -149,11 +147,10 @@ class JsIrIncrementalDataProvider(private val testServices: TestServices) : Test
val moduleCache = icCache[canonicalPath] ?: TestArtifactCache(mainModuleIr.name.asString())
val serializer = JsIrAstSerializer()
for (rebuiltFile in rebuiltFiles) {
if (rebuiltFile.first.module == mainModuleIr) {
val output = ByteArrayOutputStream()
serializer.serialize(rebuiltFile.second, output)
rebuiltFile.second.serializeTo(output)
moduleCache.binaryAsts[rebuiltFile.first.fileEntry.name] = output.toByteArray()
}
}