From 698be0fdddf51d12ceba1944eaf8532ca91252a8 Mon Sep 17 00:00:00 2001 From: Anton Bannykh Date: Sat, 7 Jan 2023 22:45:52 +0100 Subject: [PATCH] JS IR: custom JS AST serialization --- .../backend/js/ic/IncrementalCacheArtifact.kt | 13 +- .../kotlin/ir/backend/js/ic/ModuleArtifact.kt | 12 +- .../js/utils/serialization/Constants.kt | 57 ++ .../serialization/JsIrAstDeserializer.kt | 541 ++++++++++++--- .../utils/serialization/JsIrAstSerializer.kt | 651 +++++++++++++++--- .../test/utils/JsIrIncrementalDataProvider.kt | 11 +- 6 files changed, 1077 insertions(+), 208 deletions(-) create mode 100644 compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/utils/serialization/Constants.kt diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt index c9de562ec8a..c2cfdf65735 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt @@ -6,17 +6,18 @@ package org.jetbrains.kotlin.ir.backend.js.ic import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragment -import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstSerializer +import org.jetbrains.kotlin.ir.backend.js.utils.serialization.serializeTo +import org.jetbrains.kotlin.konan.file.use import java.io.BufferedOutputStream import java.io.File internal sealed class SourceFileCacheArtifact(val srcFile: KotlinSourceFile, val binaryAstFile: File) { abstract fun commitMetadata() - fun commitBinaryAst(fragment: JsIrProgramFragment, serializer: JsIrAstSerializer) { + fun commitBinaryAst(fragment: JsIrProgramFragment) { binaryAstFile.recreate() - BufferedOutputStream(binaryAstFile.outputStream()).use { bufferedOutStream -> - serializer.serialize(fragment, bufferedOutStream) + BufferedOutputStream(binaryAstFile.outputStream()).use { + fragment.serializeTo(it) } } @@ -61,12 +62,10 @@ internal class IncrementalCacheArtifact( moduleName: String, rebuiltFileFragments: Map, ): ModuleArtifact { - val serializer = JsIrAstSerializer() - val fileArtifacts = srcCacheActions.map { srcFileAction -> val rebuiltFileFragment = rebuiltFileFragments[srcFileAction.srcFile] if (rebuiltFileFragment != null) { - srcFileAction.commitBinaryAst(rebuiltFileFragment, serializer) + srcFileAction.commitBinaryAst(rebuiltFileFragment) } srcFileAction.commitMetadata() SrcFileArtifact(srcFileAction.srcFile.path, rebuiltFileFragment, srcFileAction.binaryAstFile) diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ModuleArtifact.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ModuleArtifact.kt index 2713e8e1e19..a2281e8dc20 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ModuleArtifact.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ModuleArtifact.kt @@ -8,19 +8,16 @@ package org.jetbrains.kotlin.ir.backend.js.ic import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrModule import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragment import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.safeModuleName -import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstDeserializer -import java.io.ByteArrayInputStream +import org.jetbrains.kotlin.ir.backend.js.utils.serialization.deserializeJsIrProgramFragment import java.io.File class SrcFileArtifact(val srcFilePath: String, private val fragment: JsIrProgramFragment?, private val astArtifact: File? = null) { - fun loadJsIrFragment(deserializer: JsIrAstDeserializer): JsIrProgramFragment? { + fun loadJsIrFragment(): JsIrProgramFragment? { if (fragment != null) { return fragment } return astArtifact?.ifExists { readBytes() }?.let { - ByteArrayInputStream(it).use { byteStream -> - deserializer.deserialize(byteStream) - } + deserializeJsIrProgramFragment(it) } } @@ -38,8 +35,7 @@ class ModuleArtifact( val moduleExternalName = externalModuleName ?: moduleSafeName fun loadJsIrModule(): JsIrModule { - val deserializer = JsIrAstDeserializer() - val fragments = fileArtifacts.sortedBy { it.srcFilePath }.mapNotNull { it.loadJsIrFragment(deserializer) } + val fragments = fileArtifacts.sortedBy { it.srcFilePath }.mapNotNull { it.loadJsIrFragment() } return JsIrModule(moduleSafeName, moduleExternalName, fragments) } } diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/utils/serialization/Constants.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/utils/serialization/Constants.kt new file mode 100644 index 00000000000..c3fd98e70cb --- /dev/null +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/utils/serialization/Constants.kt @@ -0,0 +1,57 @@ +/* + * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.js.utils.serialization + +// TODO: is it OK to just use UTF_8? +val SerializationCharset = Charsets.UTF_8 + +object StatementIds { + const val RETURN = 0 + const val THROW = 1 + const val BREAK = 2 + const val CONTINUE = 3 + const val DEBUGGER = 4 + const val EXPRESSION = 5 + const val VARS = 6 + const val BLOCK = 7 + const val COMPOSITE_BLOCK = 8 + const val LABEL = 9 + const val IF = 10 + const val SWITCH = 11 + const val WHILE = 12 + const val DO_WHILE = 13 + const val FOR = 14 + const val FOR_IN = 15 + const val TRY = 16 + const val EMPTY = 17 + const val SINGLE_LINE_COMMENT = 18 + const val MULTI_LINE_COMMENT = 19 +} + +object ExpressionIds { + const val THIS_REF = 0 + const val NULL = 1 + const val TRUE_LITERAL = 2 + const val FALSE_LITERAL = 3 + const val STRING_LITERAL = 4 + const val REG_EXP = 5 + const val INT_LITERAL = 6 + const val DOUBLE_LITERAL = 7 + const val ARRAY_LITERAL = 8 + const val OBJECT_LITERAL = 9 + const val FUNCTION = 10 + const val DOC_COMMENT = 11 + const val BINARY_OPERATION = 12 + const val PREFIX_OPERATION = 13 + const val POSTFIX_OPERATION = 14 + const val CONDITIONAL = 15 + const val ARRAY_ACCESS = 16 + const val NAME_REFERENCE = 17 + const val SIMPLE_NAME_REFERENCE = 18 + const val PROPERTY_REFERENCE = 19 + const val INVOCATION = 20 + const val NEW = 21 +} \ No newline at end of file diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/utils/serialization/JsIrAstDeserializer.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/utils/serialization/JsIrAstDeserializer.kt index e369e5e8423..7a75165f761 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/utils/serialization/JsIrAstDeserializer.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/utils/serialization/JsIrAstDeserializer.kt @@ -11,97 +11,454 @@ import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragmen import org.jetbrains.kotlin.ir.backend.js.utils.emptyScope import org.jetbrains.kotlin.js.backend.ast.* import org.jetbrains.kotlin.js.backend.ast.JsImportedModule -import org.jetbrains.kotlin.protobuf.CodedInputStream -import org.jetbrains.kotlin.serialization.js.ast.JsAstDeserializerBase -import org.jetbrains.kotlin.serialization.js.ast.JsAstProtoBuf.* -import java.io.InputStream +import org.jetbrains.kotlin.js.backend.ast.metadata.* +import org.jetbrains.kotlin.js.backend.ast.metadata.LocalAlias +import org.jetbrains.kotlin.js.backend.ast.metadata.SpecialFunction +import java.util.* +import java.util.ArrayDeque -class JsIrAstDeserializer : JsAstDeserializerBase() { - override val scope = emptyScope - - fun deserialize(input: InputStream): JsIrProgramFragment { - return deserialize(Chunk.parseFrom(CodedInputStream.newInstance(input).apply { setRecursionLimit(4096) })) - } - - fun deserialize(proto: Chunk): JsIrProgramFragment { - stringTable += proto.stringTable.entryList - nameTable += proto.nameTable.entryList - nameCache += nameTable.map { null } - try { - return deserialize(proto.fragment) - } finally { - stringTable.clear() - nameTable.clear() - nameCache.clear() - } - } - - private fun deserialize(proto: Fragment): JsIrProgramFragment { - val fragment = JsIrProgramFragment(proto.packageFqn) - - fragment.importedModules += proto.importedModuleList.map { importedModuleProto -> - JsImportedModule( - deserializeString(importedModuleProto.externalNameId), - deserializeName(importedModuleProto.internalNameId), - if (importedModuleProto.hasPlainReference()) deserialize(importedModuleProto.plainReference) else null - ) - } - - proto.importEntryList.associateTo(fragment.imports) { importProto -> - deserializeString(importProto.signatureId) to deserialize(importProto.expression) - } - - if (proto.hasDeclarationBlock()) { - fragment.declarations.statements += deserializeCompositeBlock(proto.declarationBlock).statements - } - if (proto.hasInitializerBlock()) { - fragment.initializers.statements += deserializeCompositeBlock(proto.initializerBlock).statements - } - if (proto.hasExportBlock()) { - fragment.exports.statements += deserializeCompositeBlock(proto.exportBlock).statements - } - if (proto.hasPolyfills()) { - fragment.polyfills.statements += deserializeCompositeBlock(proto.polyfills).statements - } - - proto.nameBindingList.associateTo(fragment.nameBindings) { nameBindingProto -> - deserializeString(nameBindingProto.signatureId) to deserializeName(nameBindingProto.nameId) - } - - proto.optionalCrossModuleImportsList.mapTo(fragment.optionalCrossModuleImports) { deserializeString(it) } - - proto.irClassModelList.associateTo(fragment.classes) { clsProto -> deserialize(clsProto) } - - if (proto.hasTestsInvocation()) { - fragment.testFunInvocation = deserialize(proto.testsInvocation) - } - - if (proto.hasMainInvocation()) { - fragment.mainFunction = deserialize(proto.mainInvocation) - } - - if (proto.hasSuiteFunction()) { - fragment.suiteFn = deserializeName(proto.suiteFunction) - } - - if (proto.hasDts()) { - fragment.dts = TypeScriptFragment(proto.dts) - } - fragment.definitions += proto.definitionsList.map { deserializeString(it) } - - return fragment - } - - private fun deserialize(proto: IrClassModel): Pair { - return deserializeName(proto.nameId) to JsIrIcClassModel(proto.superClassesList.map { deserializeName(it) }).apply { - if (proto.hasPreDeclarationBlock()) { - preDeclarationBlock.statements += deserializeCompositeBlock(proto.preDeclarationBlock).statements - } - if (proto.hasPostDeclarationBlock()) { - postDeclarationBlock.statements += deserializeCompositeBlock(proto.postDeclarationBlock).statements - } - } - } - - override fun embedSources(deserializedLocation: JsLocation, file: String): JsLocationWithSource? = null +fun deserializeJsIrProgramFragment(input: ByteArray): JsIrProgramFragment { + return JsIrAstDeserializer(input).readFragment() +} + +private class JsIrAstDeserializer(private val source: ByteArray) { + + private var offset = 0 + + private val stringTable = readArray { readString() } + private val nameTable = readArray { readName() } + + private val scope = emptyScope + private val fileStack: Deque = ArrayDeque() + + private fun readByte(): Byte { + return source[offset++] + } + + private fun readBoolean(): Boolean { + return readByte() != 0.toByte() + } + + private fun readInt(): Int { + var result = readByte().toInt() // no need to mask the highest bit + result = (result shl 8) or (readByte().toInt() and 0xFF) + result = (result shl 8) or (readByte().toInt() and 0xFF) + result = (result shl 8) or (readByte().toInt() and 0xFF) + return result + } + + private fun readLong(): Long { + var result = readByte().toLong() // no need to mask the highest bit + result = (result shl 8) or (readByte().toLong() and 0xFF) + result = (result shl 8) or (readByte().toLong() and 0xFF) + result = (result shl 8) or (readByte().toLong() and 0xFF) + result = (result shl 8) or (readByte().toLong() and 0xFF) + result = (result shl 8) or (readByte().toLong() and 0xFF) + result = (result shl 8) or (readByte().toLong() and 0xFF) + result = (result shl 8) or (readByte().toLong() and 0xFF) + return result + } + + private fun readDouble(): Double { + return Double.fromBits(readLong()) + } + + private fun readString(): String { + val length = readInt() + val result = String(source, offset, length, SerializationCharset) + offset += length + return result + } + + private inline fun readArray(readElement: () -> T): Array { + return Array(readInt()) { readElement() } + } + + private inline fun readRepeated(readElement: () -> Unit) { + var length = readInt() + while (length-- > 0) { + readElement() + } + } + + private inline fun readList(readElement: () -> T): List { + val length = readInt() + val result = ArrayList(length) + for (i in 0 until length) { + result.add(readElement()) + } + return result + } + + private inline fun ifTrue(then: () -> T): T? { + return if (readBoolean()) then() else null + } + + fun readFragment(): JsIrProgramFragment { + return JsIrProgramFragment(readString()).apply { + readRepeated { + importedModules += JsImportedModule( + externalName = stringTable[readInt()], + internalName = nameTable[readInt()], + plainReference = ifTrue { readExpression() } + ) + } + + readRepeated { imports[stringTable[readInt()]] = readExpression() } + + readRepeated { declarations.statements += readStatement() } + readRepeated { initializers.statements += readStatement() } + readRepeated { exports.statements += readStatement() } + readRepeated { polyfills.statements += readStatement() } + + readRepeated { nameBindings[stringTable[readInt()]] = nameTable[readInt()] } + readRepeated { optionalCrossModuleImports.add(stringTable[readInt()]) } + readRepeated { classes[nameTable[readInt()]] = readIrIcClassModel() } + + ifTrue { testFunInvocation = readStatement() } + ifTrue { mainFunction = readStatement() } + ifTrue { dts = TypeScriptFragment(readString()) } + ifTrue { suiteFn = nameTable[readInt()] } + + readRepeated { definitions += stringTable[readInt()] } + } + } + + private fun readIrIcClassModel(): JsIrIcClassModel { + return JsIrIcClassModel(readList { nameTable[readInt()] }).apply { + readRepeated { preDeclarationBlock.statements += readStatement() } + readRepeated { postDeclarationBlock.statements += readStatement() } + } + } + + private fun readStatement(): JsStatement { + return withComments { + withLocation { + with(StatementIds) { + when (val id = readByte().toInt()) { + RETURN -> { + JsReturn(ifTrue { readExpression() }) + } + THROW -> { + JsThrow(readExpression()) + } + BREAK -> { + JsBreak(ifTrue { JsNameRef(nameTable[readInt()]) }) + } + CONTINUE -> { + JsContinue(ifTrue { JsNameRef(nameTable[readInt()]) }) + } + DEBUGGER -> { + JsDebugger() + } + EXPRESSION -> { + JsExpressionStatement(readExpression()).apply { + ifTrue { exportedTag = stringTable[readInt()] } + } + } + VARS -> { + readVars() + } + BLOCK -> { + JsBlock().apply { + readRepeated { statements += readStatement() } + } + } + COMPOSITE_BLOCK -> { + readCompositeBlock() + } + LABEL -> { + JsLabel(nameTable[readInt()], readStatement()) + } + IF -> { + JsIf(readExpression(), readStatement(), ifTrue { readStatement() }) + } + SWITCH -> { + JsSwitch( + readExpression(), + readList { + withLocation { + ifTrue { + JsCase().apply { caseExpression = readExpression() } + } ?: JsDefault() + }.apply { + readRepeated { + statements += readStatement() + } + } + } + ) + } + WHILE -> { + JsWhile(readExpression(), readStatement()) + } + DO_WHILE -> { + JsDoWhile(readExpression(), readStatement()) + } + FOR -> { + val condition = ifTrue { readExpression() } + val incrementExpression = ifTrue { readExpression() } + val body = ifTrue { readStatement() } + + ifTrue { + JsFor( + readVars(), + condition, + incrementExpression, + body + ) + } ?: JsFor( + ifTrue { readExpression() }, + condition, + incrementExpression, + body + ) + } + FOR_IN -> { + JsForIn( + ifTrue { nameTable[readInt()] }, + ifTrue { readExpression() }, + readExpression(), + readStatement() + ) + } + TRY -> { + JsTry( + readBlock(), + readList { + JsCatch(nameTable[readInt()]).apply { + body = readBlock() + } + }, + ifTrue { readBlock() } + ) + } + EMPTY -> { + JsEmpty + } + SINGLE_LINE_COMMENT -> { + JsSingleLineComment(readString()) + } + MULTI_LINE_COMMENT -> { + JsMultiLineComment(readString()) + } + else -> error("Unknown statement id: $id") + } + } + } + }.apply { + synthetic = readBoolean() + } + } + + private val sideEffectKindValues = SideEffectKind.values() + private val jsBinaryOperatorValues = JsBinaryOperator.values() + private val jsUnaryOperatorValues = JsUnaryOperator.values() + + private fun readExpression(): JsExpression { + return withComments { + withLocation { + with(ExpressionIds) { + when (val id = readByte().toInt()) { + THIS_REF -> { + JsThisRef() + } + NULL -> { + JsNullLiteral() + } + TRUE_LITERAL -> { + JsBooleanLiteral(true) + } + FALSE_LITERAL -> { + JsBooleanLiteral(false) + } + STRING_LITERAL -> { + JsStringLiteral(stringTable[readInt()]) + } + REG_EXP -> { + JsRegExp().apply { + pattern = stringTable[readInt()] + ifTrue { flags = stringTable[readInt()] } + } + } + INT_LITERAL -> { + JsIntLiteral(readInt()) + } + DOUBLE_LITERAL -> { + JsDoubleLiteral(readDouble()) + } + ARRAY_LITERAL -> { + JsArrayLiteral(readList { readExpression() }) + } + OBJECT_LITERAL -> { + JsObjectLiteral( + readList { JsPropertyInitializer(readExpression(), readExpression()) }, + readBoolean() + ) + } + FUNCTION -> { + JsFunction(scope, readBlock(), "").apply { + readRepeated { parameters += readParameter() } + ifTrue { name = nameTable[readInt()] } + isLocal = readBoolean() + } + } + DOC_COMMENT -> { + val tags = hashMapOf() + readRepeated { + tags[stringTable[readInt()]] = ifTrue { readExpression() } ?: stringTable[readInt()] + } + JsDocComment(tags) + } + BINARY_OPERATION -> { + JsBinaryOperation( + jsBinaryOperatorValues[readInt()], + readExpression(), + readExpression() + ) + } + PREFIX_OPERATION -> { + JsPrefixOperation(jsUnaryOperatorValues[readInt()], readExpression()) + } + POSTFIX_OPERATION -> { + JsPostfixOperation(jsUnaryOperatorValues[readInt()], readExpression()) + } + CONDITIONAL -> { + JsConditional( + readExpression(), + readExpression(), + readExpression() + ) + } + ARRAY_ACCESS -> { + JsArrayAccess(readExpression(), readExpression()) + } + NAME_REFERENCE -> { + JsNameRef(nameTable[readInt()]).apply { + ifTrue { qualifier = readExpression() } + ifTrue { isInline = readBoolean() } + } + } + SIMPLE_NAME_REFERENCE -> { + JsNameRef(nameTable[readInt()]) + } + PROPERTY_REFERENCE -> { + JsNameRef(stringTable[readInt()]).apply { + ifTrue { qualifier = readExpression() } + ifTrue { isInline = readBoolean() } + } + } + INVOCATION -> { + JsInvocation(readExpression(), readList { readExpression() }).apply { + ifTrue { isInline = readBoolean() } + } + } + NEW -> { + JsNew(readExpression(), readList { readExpression() }) + } + else -> error("Unknown expression id: $id") + } + } + } + }.apply { + synthetic = readBoolean() + sideEffects = sideEffectKindValues[readInt()] + ifTrue { localAlias = readJsImportedModule() } + } + } + + private fun readJsImportedModule(): JsImportedModule { + return JsImportedModule( + stringTable[readInt()], + nameTable[readInt()], + ifTrue { readExpression() } + ) + } + + private fun readParameter(): JsParameter { + return JsParameter(nameTable[readInt()]).apply { + hasDefaultValue = readBoolean() + } + } + + private fun readCompositeBlock(): JsCompositeBlock { + return JsCompositeBlock().apply { + readRepeated { statements += readStatement() } + } + } + + private fun readBlock(): JsBlock { + return ifTrue { readCompositeBlock() } ?: JsBlock().apply { + readRepeated { statements += readStatement() } + } + } + + private fun readVars(): JsVars { + return JsVars(readBoolean()).apply { + readRepeated { + vars += withLocation { + JsVars.JsVar(nameTable[readInt()], ifTrue { readExpression() }) + } + } + ifTrue { exportedPackage = stringTable[readInt()] } + } + } + + private val specialFunctionValues = SpecialFunction.values() + + private fun readName(): JsName { + val identifier = stringTable[readInt()] + val name = ifTrue { + JsScope.declareTemporaryName(identifier) + } ?: JsDynamicScope.declareName(identifier) + ifTrue { name.localAlias = readLocalAlias() } + ifTrue { name.imported = true } + ifTrue { name.specialFunction = specialFunctionValues[readInt()] } + return name + } + + private fun readLocalAlias(): LocalAlias { + return LocalAlias( + nameTable[readInt()], + ifTrue { stringTable[readInt()] } + ) + } + + private fun readComment(): JsComment { + val text = readString() + return ifTrue { JsMultiLineComment(text) } ?: JsSingleLineComment(text) + } + + private inline fun withLocation(action: () -> T): T { + return ifTrue { + val deserializedFile = ifTrue { stringTable[readInt()] } + val file = deserializedFile ?: fileStack.peek() + + val startLine = readInt() + val startChar = readInt() + val deserializedLocation = file?.let { JsLocation(it, startLine, startChar) } + + val shouldUpdateFile = deserializedFile != null && deserializedFile != fileStack.peek() + + if (shouldUpdateFile) { + fileStack.push(deserializedFile) + } + val node = action() + if (deserializedLocation != null) { + node.source = deserializedLocation + } + if (shouldUpdateFile) { + fileStack.pop() + } + + node + } ?: action() + } + + private inline fun withComments(action: () -> T): T { + return action().apply { + ifTrue { this.commentsBeforeNode = readArray { readComment() }.toList() } + ifTrue { this.commentsAfterNode = readArray { readComment() }.toList() } + } + } } diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/utils/serialization/JsIrAstSerializer.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/utils/serialization/JsIrAstSerializer.kt index 0219bc349a0..b2606e897eb 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/utils/serialization/JsIrAstSerializer.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/utils/serialization/JsIrAstSerializer.kt @@ -5,114 +5,577 @@ package org.jetbrains.kotlin.ir.backend.js.utils.serialization -import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrClassModel import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrIcClassModel import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragment import org.jetbrains.kotlin.js.backend.ast.* -import org.jetbrains.kotlin.serialization.js.ast.JsAstProtoBuf.* -import org.jetbrains.kotlin.serialization.js.ast.JsAstSerializerBase -import org.jetbrains.kotlin.utils.addToStdlib.ifNotEmpty +import org.jetbrains.kotlin.js.backend.ast.JsImportedModule +import org.jetbrains.kotlin.js.backend.ast.metadata.* import org.jetbrains.kotlin.utils.addToStdlib.safeAs +import java.io.ByteArrayOutputStream +import java.io.DataOutputStream import java.io.OutputStream +import java.util.* +import java.util.ArrayDeque -class JsIrAstSerializer: JsAstSerializerBase() { +fun JsIrProgramFragment.serializeTo(output: OutputStream) { + JsIrAstSerializer().append(this).saveTo(output) +} - fun serialize(fragment: JsIrProgramFragment, output: OutputStream) { - importedNames.clear() - importedNames += fragment.imports.map { fragment.nameBindings[it.key]!! } - serialize(fragment).writeTo(output) - } +private class DataWriter { - fun serialize(fragment: JsIrProgramFragment): Chunk { - try { - val chunkBuilder = Chunk.newBuilder() - chunkBuilder.fragment = serializeFragment(fragment) - chunkBuilder.nameTable = nameTableBuilder.build() - chunkBuilder.stringTable = stringTableBuilder.build() - return chunkBuilder.build() - } finally { - nameTableBuilder.clear() - stringTableBuilder.clear() - nameMap.clear() - stringMap.clear() + val data = ByteArrayOutputStream() + private val output = DataOutputStream(data) + + fun saveTo(output: DataOutputStream) { + output.let { + data.writeTo(it) } } - private fun serializeFragment(fragment: JsIrProgramFragment): Fragment { - val fragmentBuilder = Fragment.newBuilder() - - fragmentBuilder.packageFqn = fragment.packageFqn - - for (importedModule in fragment.importedModules) { - val importedModuleBuilder = ImportedModule.newBuilder() - importedModuleBuilder.externalNameId = serialize(importedModule.externalName) - importedModuleBuilder.internalNameId = serialize(importedModule.internalName) - importedModule.plainReference?.let { importedModuleBuilder.plainReference = serialize(it) } - fragmentBuilder.addImportedModule(importedModuleBuilder) - } - - for ((signature, expression) in fragment.imports) { - val importBuilder = Import.newBuilder() - importBuilder.signatureId = serialize(signature) - importBuilder.expression = serialize(expression) - fragmentBuilder.addImportEntry(importBuilder) - } - - fragmentBuilder.declarationBlock = serializeBlock(fragment.declarations) - fragmentBuilder.initializerBlock = serializeBlock(fragment.initializers) - fragmentBuilder.exportBlock = serializeBlock(fragment.exports) - fragmentBuilder.polyfills = serializeBlock(fragment.polyfills) - - for ((key, name) in fragment.nameBindings.entries) { - val nameBindingBuilder = NameBinding.newBuilder() - nameBindingBuilder.signatureId = serialize(key) - nameBindingBuilder.nameId = serialize(name) - fragmentBuilder.addNameBinding(nameBindingBuilder) - } - - fragment.optionalCrossModuleImports.forEach { - fragmentBuilder.addOptionalCrossModuleImports(serialize(it)) - } - - fragment.classes.entries.forEach { (name, model) -> fragmentBuilder.addIrClassModel(serialize(name, model)) } - - fragment.testFunInvocation?.let { - fragmentBuilder.setTestsInvocation(serialize(it)) - } - - fragment.mainFunction?.let { - fragmentBuilder.setMainInvocation(serialize(it)) - } - - fragment.dts?.let { - fragmentBuilder.dts = it.raw - } - - fragment.suiteFn?.let { - fragmentBuilder.setSuiteFunction(serialize(it)) - } - - fragment.definitions.forEach { - fragmentBuilder.addDefinitions(serialize(it)) - } - - return fragmentBuilder.build() + fun writeByte(byte: Int) { + // Limit bytes to positive values to avoid conversion in deserializer + if ((byte and 0x7F.inv()) != 0) error("Byte out of bounds: $byte") + output.writeByte(byte) } - private fun serialize(name: JsName, classModel: JsIrIcClassModel): IrClassModel { - val builder = IrClassModel.newBuilder() - builder.nameId = serialize(name) - classModel.superClasses.forEach { builder.addSuperClasses(serialize(it)) } - if (classModel.preDeclarationBlock.statements.isNotEmpty()) { - builder.preDeclarationBlock = serializeBlock(classModel.preDeclarationBlock) - } - if (classModel.postDeclarationBlock.statements.isNotEmpty()) { - builder.postDeclarationBlock = serializeBlock(classModel.postDeclarationBlock) - } - return builder.build() + fun writeByteArray(byteArray: ByteArray) { + output.writeInt(byteArray.size) + output.write(byteArray) } - override fun extractLocation(node: JsNode): JsLocation? { - return node.source.safeAs()?.asSimpleLocation() + fun writeString(string: String) { + writeByteArray(string.toByteArray(SerializationCharset)) + } + + fun writeBoolean(boolean: Boolean) { + output.writeBoolean(boolean) + } + + fun writeInt(int: Int) { + output.writeInt(int) + } + + fun writeDouble(double: Double) { + output.writeDouble(double) + } + + inline fun writeCollection(collection: Collection, writeItem: (T) -> Unit) { + output.writeInt(collection.size) + collection.forEach(writeItem) + } + + inline fun ifNotNull(t: T?, write: (T) -> Unit): T? { + output.writeBoolean(t != null) + if (t != null) { + write(t) + } + return t + } + + inline fun ifTrue(condition: Boolean, write: () -> Unit) { + output.writeBoolean(condition) + if (condition) { + write() + } + } +} + +private class JsIrAstSerializer { + + private val fragmentSerializer = DataWriter() + private val nameSerializer = DataWriter() + private val stringSerializer = DataWriter() + + private val nameMap = mutableMapOf() + private val stringMap = mutableMapOf() + private val fileStack: Deque = ArrayDeque() + private val importedNames = mutableSetOf() + + fun append(fragment: JsIrProgramFragment): JsIrAstSerializer { + importedNames += fragment.imports.map { fragment.nameBindings[it.key]!! } + fragmentSerializer.writeFragment(fragment) + return this + } + + fun saveTo(rawOutput: OutputStream) { + DataOutputStream(rawOutput).use { + it.writeInt(stringMap.size) + stringSerializer.saveTo(it) + + it.writeInt(nameMap.size) + nameSerializer.saveTo(it) + + fragmentSerializer.saveTo(it) + } + } + + private fun DataWriter.writeFragment(fragment: JsIrProgramFragment) { + writeString(fragment.packageFqn) + + writeCollection(fragment.importedModules) { + writeInt(internalizeString(it.externalName)) + writeInt(internalizeName(it.internalName)) + ifNotNull(it.plainReference) { + writeExpression(it) + } + } + + writeCollection(fragment.imports.entries) { (signatureId, expression) -> + writeInt(internalizeString(signatureId)) + writeExpression(expression) + } + + writeCompositeBlock(fragment.declarations) + writeCompositeBlock(fragment.initializers) + writeCompositeBlock(fragment.exports) + writeCompositeBlock(fragment.polyfills) + + writeCollection(fragment.nameBindings.entries) { (key, name) -> + writeInt(internalizeString(key)) + writeInt(internalizeName(name)) + } + + writeCollection(fragment.optionalCrossModuleImports) { + writeInt(internalizeString(it)) + } + + writeCollection(fragment.classes.entries) { (name, model) -> + writeInt(internalizeName(name)) + writeIrIcModel(model) + } + + ifNotNull(fragment.testFunInvocation) { + writeStatement(it) + } + + ifNotNull(fragment.mainFunction) { + writeStatement(it) + } + + ifNotNull(fragment.dts) { + writeString(it.raw) + } + + ifNotNull(fragment.suiteFn) { + writeInt(internalizeName(it)) + } + + writeCollection(fragment.definitions) { + writeInt(internalizeString(it)) + } + } + + private fun DataWriter.writeIrIcModel(classModel: JsIrIcClassModel) { + writeCollection(classModel.superClasses) { + writeInt(internalizeName(it)) + } + writeCompositeBlock(classModel.preDeclarationBlock) + writeCompositeBlock(classModel.postDeclarationBlock) + } + + private fun DataWriter.writeStatement(statement: JsStatement) { + val visitor = object : JsVisitor() { + override fun visitReturn(x: JsReturn) { + writeByte(StatementIds.RETURN) + ifNotNull(x.expression) { + writeExpression(it) + } + } + + override fun visitThrow(x: JsThrow) { + writeByte(StatementIds.THROW) + writeExpression(x.expression) + } + + override fun visitBreak(x: JsBreak) { + writeByte(StatementIds.BREAK) + ifNotNull(x.label) { + writeInt(internalizeName(it.name!!)) + } + } + + override fun visitContinue(x: JsContinue) { + writeByte(StatementIds.CONTINUE) + ifNotNull(x.label) { + writeInt(internalizeName(it.name!!)) + } + } + + override fun visitDebugger(x: JsDebugger) { + writeByte(StatementIds.DEBUGGER) + } + + override fun visitExpressionStatement(x: JsExpressionStatement) { + writeByte(StatementIds.EXPRESSION) + writeExpression(x.expression) + ifNotNull(x.exportedTag) { writeInt(internalizeString(it)) } + } + + override fun visitVars(x: JsVars) { + writeByte(StatementIds.VARS) + writeVars(x) + } + + override fun visitBlock(x: JsBlock) { + if (x is JsCompositeBlock) { + writeByte(StatementIds.COMPOSITE_BLOCK) + writeCompositeBlock(x) + } else { + writeByte(StatementIds.BLOCK) + writeCollection(x.statements) { + writeStatement(it) + } + } + } + + override fun visitLabel(x: JsLabel) { + writeByte(StatementIds.LABEL) + writeInt(internalizeName(x.name)) + writeStatement(x.statement) + } + + override fun visitIf(x: JsIf) { + writeByte(StatementIds.IF) + writeExpression(x.ifExpression) + writeStatement(x.thenStatement) + ifNotNull(x.elseStatement) { writeStatement(it) } + } + + override fun visit(x: JsSwitch) { + writeByte(StatementIds.SWITCH) + writeExpression(x.expression) + writeCollection(x.cases) { case -> + withLocation(case) { + ifNotNull(case as? JsCase) { + writeExpression(it.caseExpression) + } + } + writeCollection(case.statements) { part -> + writeStatement(part) + } + } + } + + override fun visitWhile(x: JsWhile) { + writeByte(StatementIds.WHILE) + writeExpression(x.condition) + writeStatement(x.body) + } + + override fun visitDoWhile(x: JsDoWhile) { + writeByte(StatementIds.DO_WHILE) + writeExpression(x.condition) + writeStatement(x.body) + } + + override fun visitFor(x: JsFor) { + writeByte(StatementIds.FOR) + ifNotNull(x.condition) { writeExpression(it) } + ifNotNull(x.incrementExpression) { writeExpression(it) } + ifNotNull(x.body) { writeStatement(it) } + + ifNotNull(x.initVars) { + writeVars(it) + } ?: ifNotNull(x.initExpression) { + writeExpression(it) + } + } + + override fun visitForIn(x: JsForIn) { + writeByte(StatementIds.FOR_IN) + ifNotNull(x.iterVarName) { + writeInt(internalizeName(it)) + } + ifNotNull(x.iterExpression) { writeExpression(it) } + writeExpression(x.objectExpression) + writeStatement(x.body) + } + + override fun visitTry(x: JsTry) { + writeByte(StatementIds.TRY) + writeBlock(x.tryBlock) + writeCollection(x.catches) { c -> + writeInt(internalizeName(c.parameter.name)) + writeBlock(c.body) + } + ifNotNull(x.finallyBlock) { writeBlock(it) } + } + + override fun visitEmpty(x: JsEmpty) { + writeByte(StatementIds.EMPTY) + } + + override fun visitSingleLineComment(comment: JsSingleLineComment) { + writeByte(StatementIds.SINGLE_LINE_COMMENT) + writeString(comment.text) + } + + override fun visitMultiLineComment(comment: JsMultiLineComment) { + writeByte(StatementIds.MULTI_LINE_COMMENT) + writeString(comment.text) + } + + override fun visitElement(node: JsNode) { + error("Unknown statement type: ${statement::class.qualifiedName}") + } + } + + withComments(statement) { + withLocation(statement) { + statement.accept(visitor) + } + } + + writeBoolean((statement as HasMetadata).synthetic) + } + + private fun DataWriter.writeExpression(expression: JsExpression) { + val visitor = object : JsVisitor() { + + override fun visitThis(x: JsThisRef) { + writeByte(ExpressionIds.THIS_REF) + } + + override fun visitNull(x: JsNullLiteral) { + writeByte(ExpressionIds.NULL) + } + + override fun visitBoolean(x: JsBooleanLiteral) { + if (x.value) { + writeByte(ExpressionIds.TRUE_LITERAL) + } else { + writeByte(ExpressionIds.FALSE_LITERAL) + } + } + + override fun visitString(x: JsStringLiteral) { + writeByte(ExpressionIds.STRING_LITERAL) + writeInt(internalizeString(x.value)) + } + + override fun visitRegExp(x: JsRegExp) { + writeByte(ExpressionIds.REG_EXP) + writeInt(internalizeString(x.pattern)) + ifNotNull(x.flags) { writeInt(internalizeString(it)) } + } + + override fun visitInt(x: JsIntLiteral) { + writeByte(ExpressionIds.INT_LITERAL) + writeInt(x.value) + } + + override fun visitDouble(x: JsDoubleLiteral) { + writeByte(ExpressionIds.DOUBLE_LITERAL) + writeDouble(x.value) + } + + override fun visitArray(x: JsArrayLiteral) { + writeByte(ExpressionIds.ARRAY_LITERAL) + writeCollection(x.expressions) { + writeExpression(it) + } + } + + override fun visitObjectLiteral(x: JsObjectLiteral) { + writeByte(ExpressionIds.OBJECT_LITERAL) + writeCollection(x.propertyInitializers) { + writeExpression(it.labelExpr) + writeExpression(it.valueExpr) + } + writeBoolean(x.isMultiline) + } + + override fun visitFunction(x: JsFunction) { + writeByte(ExpressionIds.FUNCTION) + writeBlock(x.body) + writeCollection(x.parameters) { writeParameter(it) } + ifNotNull(x.name) { + writeInt(internalizeName(it)) + } + writeBoolean(x.isLocal) + } + + override fun visitDocComment(comment: JsDocComment) { + writeByte(ExpressionIds.DOC_COMMENT) + writeCollection(comment.tags.entries) { (name, value) -> + writeInt(internalizeString(name)) + + ifNotNull(value as? JsNameRef) { + writeExpression(it) + } ?: if (value is String) { + writeInt(internalizeString(value)) + } else { + error("Unsupported tag: ${value::class.qualifiedName}") + } + } + } + + override fun visitBinaryExpression(x: JsBinaryOperation) { + writeByte(ExpressionIds.BINARY_OPERATION) + writeInt(x.operator.ordinal) + writeExpression(x.arg1) + writeExpression(x.arg2) + } + + override fun visitPrefixOperation(x: JsPrefixOperation) { + writeByte(ExpressionIds.PREFIX_OPERATION) + writeInt(x.operator.ordinal) + writeExpression(x.arg) + } + + override fun visitPostfixOperation(x: JsPostfixOperation) { + writeByte(ExpressionIds.POSTFIX_OPERATION) + writeInt(x.operator.ordinal) + writeExpression(x.arg) + } + + override fun visitConditional(x: JsConditional) { + writeByte(ExpressionIds.CONDITIONAL) + writeExpression(x.testExpression) + writeExpression(x.thenExpression) + writeExpression(x.elseExpression) + } + + override fun visitArrayAccess(x: JsArrayAccess) { + writeByte(ExpressionIds.ARRAY_ACCESS) + writeExpression(x.arrayExpression) + writeExpression(x.indexExpression) + } + + override fun visitNameRef(nameRef: JsNameRef) { + val name = nameRef.name + val qualifier = nameRef.qualifier + if (name != null) { + if (qualifier != null || nameRef.isInline == true) { + writeByte(ExpressionIds.NAME_REFERENCE) + writeInt(internalizeName(name)) + ifNotNull(qualifier) { writeExpression(it) } + ifNotNull(nameRef.isInline) { writeBoolean(it) } + } else { + writeByte(ExpressionIds.SIMPLE_NAME_REFERENCE) + writeInt(internalizeName(name)) + } + } else { + writeByte(ExpressionIds.PROPERTY_REFERENCE) + writeInt(internalizeString(nameRef.ident)) + ifNotNull(qualifier) { writeExpression(it) } + ifNotNull(nameRef.isInline) { writeBoolean(it) } + } + } + + override fun visitInvocation(invocation: JsInvocation) { + writeByte(ExpressionIds.INVOCATION) + writeExpression(invocation.qualifier) + writeCollection(invocation.arguments) { writeExpression(it) } + ifNotNull(invocation.isInline) { writeBoolean(it) } + } + + override fun visitNew(x: JsNew) { + writeByte(ExpressionIds.NEW) + writeExpression(x.constructorExpression) + writeCollection(x.arguments) { writeExpression(it) } + } + } + + withComments(expression) { + withLocation(expression) { + expression.accept(visitor) + } + } + + writeBoolean(expression.synthetic) + writeInt(expression.sideEffects.ordinal) + ifNotNull(expression.localAlias) { writeImportedModule(it) } + } + + private fun DataWriter.writeImportedModule(module: JsImportedModule) { + writeInt(internalizeString(module.externalName)) + writeInt(internalizeName(module.internalName)) + ifNotNull(module.plainReference) { writeExpression(it) } + } + + private fun DataWriter.writeParameter(parameter: JsParameter) { + writeInt(internalizeName(parameter.name)) + writeBoolean(parameter.hasDefaultValue) + } + + private fun DataWriter.writeCompositeBlock(block: JsCompositeBlock) { + writeCollection(block.statements) { writeStatement(it) } + } + + private fun DataWriter.writeBlock(block: JsBlock) { + writeBoolean(block is JsCompositeBlock) + writeCollection(block.statements) { writeStatement(it) } + } + + private fun DataWriter.writeVars(vars: JsVars) { + writeBoolean(vars.isMultiline) + writeCollection(vars.vars) { varDecl -> + withLocation(varDecl) { + writeInt(internalizeName(varDecl.name)) + ifNotNull(varDecl.initExpression) { writeExpression(it) } + } + } + ifNotNull(vars.exportedPackage) { writeInt(internalizeString(it)) } + } + + private fun internalizeName(name: JsName): Int = nameMap.getOrPut(name) { + // Make sure all dependant JsName's are already serialized. + name.localAlias?.let { internalizeName(it.name) } + + nameSerializer.apply { + writeInt(internalizeString(name.ident)) + writeBoolean(name.isTemporary) + ifNotNull(name.localAlias) { writeLocalAlias(it) } + writeBoolean(name.imported && name !in importedNames) + ifNotNull(name.specialFunction) { writeInt(it.ordinal) } + } + nameMap.size + } + + private fun DataWriter.writeLocalAlias(alias: LocalAlias) { + writeInt(internalizeName(alias.name)) + ifNotNull(alias.tag) { writeInt(internalizeString(it)) } + } + + private fun internalizeString(string: String) = stringMap.getOrPut(string) { + stringSerializer.writeString(string) + stringMap.size + } + + private fun DataWriter.writeComment(comment: JsComment) { + writeString(comment.text) + when (comment) { + is JsSingleLineComment -> writeBoolean(false) + is JsMultiLineComment -> writeBoolean(true) + else -> error("Unknown type of comment ${comment.javaClass.name}") + } + } + + private inline fun DataWriter.withLocation(node: JsNode, inner: () -> Unit) { + val location = node.source.safeAs()?.asSimpleLocation() + ifNotNull(location) { + val lastFile = fileStack.peek() + val newFile = it.file + val fileChanged = lastFile != newFile + ifTrue(fileChanged) { + writeInt(internalizeString(newFile)) + fileStack.push(it.file) + } + writeInt(it.startLine) + writeInt(it.startChar) + + inner() + + if (fileChanged) { + fileStack.pop() + } + } ?: inner() + } + + private inline fun DataWriter.withComments(node: JsNode, inner: () -> Unit) { + inner() + ifNotNull(node.commentsBeforeNode) { comments -> writeCollection(comments) { writeComment(it) } } + ifNotNull(node.commentsAfterNode) { comments -> writeCollection(comments) { writeComment(it) } } } } diff --git a/js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt b/js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt index 675e35fb77c..b890ea257b1 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt @@ -9,8 +9,8 @@ import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.ir.backend.js.WholeWorldStageController import org.jetbrains.kotlin.ir.backend.js.ic.* import org.jetbrains.kotlin.ir.backend.js.moduleName -import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstDeserializer -import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstSerializer +import org.jetbrains.kotlin.ir.backend.js.utils.serialization.serializeTo +import org.jetbrains.kotlin.ir.backend.js.utils.serialization.deserializeJsIrProgramFragment import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC import org.jetbrains.kotlin.js.test.handlers.JsBoxRunner import org.jetbrains.kotlin.konan.properties.propertyList @@ -24,13 +24,11 @@ import org.jetbrains.kotlin.test.services.TestServices import org.jetbrains.kotlin.test.services.compilerConfigurationProvider import org.jetbrains.kotlin.test.services.configuration.JsEnvironmentConfigurator import org.jetbrains.kotlin.test.services.jsLibraryProvider -import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream import java.io.File private class TestArtifactCache(val moduleName: String, val binaryAsts: MutableMap = mutableMapOf()) { fun fetchArtifacts(): ModuleArtifact { - val deserializer = JsIrAstDeserializer() return ModuleArtifact( moduleName = moduleName, fileArtifacts = binaryAsts.entries.map { @@ -39,7 +37,7 @@ private class TestArtifactCache(val moduleName: String, val binaryAsts: MutableM // TODO: It will be better to use saved fragments, but it doesn't work // Merger.merge() + JsNode.resolveTemporaryNames() modify fragments, // therefore the sequential calls produce different results - fragment = deserializer.deserialize(ByteArrayInputStream(it.value)) + fragment = deserializeJsIrProgramFragment(it.value) ) } ) @@ -149,11 +147,10 @@ class JsIrIncrementalDataProvider(private val testServices: TestServices) : Test val moduleCache = icCache[canonicalPath] ?: TestArtifactCache(mainModuleIr.name.asString()) - val serializer = JsIrAstSerializer() for (rebuiltFile in rebuiltFiles) { if (rebuiltFile.first.module == mainModuleIr) { val output = ByteArrayOutputStream() - serializer.serialize(rebuiltFile.second, output) + rebuiltFile.second.serializeTo(output) moduleCache.binaryAsts[rebuiltFile.first.fileEntry.name] = output.toByteArray() } }