Enhanced support for custom serializers definition

kotlin.Triple serializer

Resolving serializers for mutable collection interfaces on JS

Remove idea-related source files form maven compilation

Serialization url in plugin.xml

Removed note about JPS incompatibility

Don't use serial names in produced JS code (https://github.com/Kotlin/kotlinx.serialization/issues/43)

Generator for calling `update` functions

Fix synthetic serialinfo implementations for 2-slots vars (double and long) (https://github.com/Kotlin/kotlinx.serialization/issues/60)

Prohibit usage of nullable serializers for non-null fields (https://github.com/Kotlin/kotlinx.serialization/issues/59)

Fix incorrect `this` referencing after update to 1.2.20 compiler

libraries/tools/kotlin-maven-serialization/pom.xml

@@ -7,9 +7,11 @@
     <properties>
         <maven.version>3.0.5</maven.version>
         <serialization.src>${basedir}/../../../plugins/kotlin-serialization/kotlin-serialization-compiler/src/</serialization.src>
+        <serialization.src.kt>${basedir}/../../../plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler</serialization.src.kt>
         <serialization.maven.plugin.src>${basedir}/src/main/kotlin</serialization.maven.plugin.src>
         <serialization.maven.plugin.resources>${basedir}/src/main/resources</serialization.maven.plugin.resources>
         <serialization.target-src>${basedir}/target/src/main/kotlin</serialization.target-src>
+        <serialization.target-src.kt>${basedir}/target/src/main/kotlin/org/jetbrains/kotlinx/serialization/compiler</serialization.target-src.kt>
         <serialization.target-resources>${basedir}/target/resource</serialization.target-resources>
         <kotlin.version>1.2-SNAPSHOT</kotlin.version>
     </properties>
@@ -17,7 +19,7 @@
     <groupId>org.jetbrains.kotlinx</groupId>
     <artifactId>kotlinx-maven-serialization-plugin</artifactId>
     <packaging>jar</packaging>
-    <version>0.2</version>
+    <version>0.3</version>
     <description>Serialization plugin for Maven</description>
 
     <repositories>
@@ -56,7 +58,7 @@
                 <version>3.0.0</version>
                 <executions>
                     <execution>
-                        <id>copy-sources</id>
+                        <id>copy-maven-sources</id>
                         <phase>validate</phase>
                         <goals>
                             <goal>copy-resources</goal>
@@ -64,11 +66,23 @@
                         <configuration>
                             <outputDirectory>${serialization.target-src}</outputDirectory>
                             <resources>
-                                <resource><directory>${serialization.src}</directory></resource>
                                 <resource><directory>${serialization.maven.plugin.src}</directory></resource>
                             </resources>
                         </configuration>
                     </execution>
+                    <execution>
+                        <id>copy-sources</id>
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${serialization.target-src.kt}</outputDirectory>
+                            <resources>
+                                <resource><directory>${serialization.src.kt}</directory></resource>
+                            </resources>
+                        </configuration>
+                    </execution>
                     <execution>
                         <id>copy-resources</id>
                         <phase>validate</phase>

libraries/tools/kotlin-serialization/build.gradle

@@ -8,7 +8,7 @@ configureJvmProject(project)
 configurePublishing(project)
 
 group = 'org.jetbrains.kotlinx'
-version = '0.2'
+version = '0.3'
 if (!project.hasProperty("deploy")) {
     version = "$version-SNAPSHOT"
 }

plugins/kotlin-serialization/kotlin-serialization-compiler/README.md

@@ -1,12 +1,7 @@
 # Kotlin serialization IDEA plugin
 
 Kotlin serialization plugin consists of three parts: a compiler plugin, an IntelliJ plugin and a runtime library.
-This is the IDEA plugin. Gradle and Maven plugins can be found in `libraries` folder.
+This is the folder with common source for all plugins, IDEA plugin is built from here. Gradle and Maven plugins can be found in `libraries` folder.
-
-Please note that this plugin currently works only for highlighting and resolving symbols, and it doesn't work with embedded IDEA compiler.
-To build any project with serialization within IDEA, you'll need to delegate all build actions to gradle:
-`File - Settings - Build, Execution, Deployment - Build Tools - Gradle - Runner -` tick `Delegate IDE build/run actions to gradle`.
-IntelliJ projects currently are not supported.
 
 Compiler plugins are uploaded on bintray: https://bintray.com/kotlin/kotlinx/kotlinx.serialization.plugin
 

plugins/kotlin-serialization/kotlin-serialization-compiler/src/META-INF/plugin.xml

@@ -1,7 +1,7 @@
-<idea-plugin>
+<idea-plugin url="https://github.com/Kotlin/kotlinx.serialization" >
     <name>Kotlin-serialization</name>
     <id>org.jetbrains.kotlinx.serialization</id>
-    <version>0.2</version>
+    <version>0.3</version>
 
     <depends>org.jetbrains.kotlin</depends>
     <idea-version since-build="172"/>

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/backend/common/TypeUtil.kt

@@ -90,10 +90,12 @@ fun findStandardKotlinTypeSerializer(module: ModuleDescriptor, kType: KotlinType
         "C", "kotlin.Char" -> "CharSerializer"
         "kotlin.String" -> "StringSerializer"
         "kotlin.Pair" -> "PairSerializer"
-        "kotlin.collections.Collection", "kotlin.collections.List", "kotlin.collections.ArrayList" -> "ArrayListSerializer"
+        "kotlin.Triple" -> "TripleSerializer"
-        "kotlin.collections.Set", "kotlin.collections.LinkedHashSet" -> "LinkedHashSetSerializer"
+        "kotlin.collections.Collection", "kotlin.collections.List",
+        "kotlin.collections.ArrayList", "kotlin.collections.MutableList" -> "ArrayListSerializer"
+        "kotlin.collections.Set", "kotlin.collections.LinkedHashSet", "kotlin.collections.MutableSet" -> "LinkedHashSetSerializer"
         "kotlin.collections.HashSet" -> "HashSetSerializer"
-        "kotlin.collections.Map", "kotlin.collections.LinkedHashMap" -> "LinkedHashMapSerializer"
+        "kotlin.collections.Map", "kotlin.collections.LinkedHashMap", "kotlin.collections.MutableMap" -> "LinkedHashMapSerializer"
         "kotlin.collections.HashMap" -> "HashMapSerializer"
         "kotlin.collections.Map.Entry" -> "MapEntrySerializer"
         else -> return null

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/backend/js/JsCodegenUtil.kt

@@ -20,6 +20,7 @@ import org.jetbrains.kotlin.descriptors.FunctionDescriptor
 import org.jetbrains.kotlin.js.backend.ast.*
 import org.jetbrains.kotlin.js.translate.context.TranslationContext
 import org.jetbrains.kotlin.js.translate.expression.translateAndAliasParameters
+import org.jetbrains.kotlin.js.translate.utils.JsAstUtils
 
 internal class JsBlockBuilder {
     val block: JsBlock = JsBlock()
@@ -81,4 +82,13 @@ internal fun TranslationContext.buildFunction(descriptor: FunctionDescriptor, bo
     b.bodyGen(functionObject, innerCtx)
     functionObject.body.statements += b.body
     return functionObject
-}
+}
+
+internal fun propNotSeenTest(seenVar: JsNameRef, index: Int): JsBinaryOperation = JsAstUtils.equality(
+        JsBinaryOperation(
+                JsBinaryOperator.BIT_AND,
+                seenVar,
+                JsIntLiteral(1 shl (index % 32))
+        ),
+        JsIntLiteral(0)
+)

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/backend/js/SerializableJsTranslator.kt

@@ -25,10 +25,8 @@ import org.jetbrains.kotlin.js.translate.context.Namer
 import org.jetbrains.kotlin.js.translate.context.TranslationContext
 import org.jetbrains.kotlin.js.translate.declaration.DeclarationBodyVisitor
 import org.jetbrains.kotlin.js.translate.general.Translation
-import org.jetbrains.kotlin.js.translate.utils.JsAstUtils
 import org.jetbrains.kotlin.js.translate.utils.TranslationUtils
 import org.jetbrains.kotlin.psi.KtExpression
-import org.jetbrains.kotlin.psi.KtProperty
 import org.jetbrains.kotlin.psi.KtPureClassOrObject
 import org.jetbrains.kotlinx.serialization.compiler.backend.common.SerializableCodegen
 import org.jetbrains.kotlinx.serialization.compiler.backend.common.anonymousInitializers
@@ -53,7 +51,7 @@ class SerializableJsTranslator(val declaration: KtPureClassOrObject,
 
         val f = context.buildFunction(constructorDescriptor) { jsFun, context ->
             val thiz = jsFun.scope.declareName(Namer.ANOTHER_THIS_PARAMETER_NAME).makeRef()
-            val context = context.innerContextWithAliased(serializableDescriptor, thiz)
+            val context = context.innerContextWithAliased(serializableDescriptor.thisAsReceiverParameter, thiz)
 
             +JsVars(JsVars.JsVar(thiz.name, Namer.createObjectWithPrototypeFrom(context.getQualifiedReference(serializableDescriptor))))
             val seenVar = jsFun.parameters[0].name.makeRef()
@@ -71,14 +69,7 @@ class SerializableJsTranslator(val declaration: KtPureClassOrObject,
                     JsThrow(JsNew(missingExceptionClassRef, listOf(JsStringLiteral(prop.name))))
                 }
                 // (seen & 1 << i == 0) -- not seen
-                val notSeenTest = JsAstUtils.equality(
+                val notSeenTest = propNotSeenTest(seenVar, index)
-                        JsBinaryOperation(
-                                JsBinaryOperator.BIT_AND,
-                                seenVar,
-                                JsIntLiteral(1 shl (index % 32))
-                        ),
-                        JsIntLiteral(0)
-                )
                 +JsIf(notSeenTest, ifNotSeenStmt, assignParamStmt)
             }
 

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/backend/js/SerializerJsTranslator.kt

@@ -228,7 +228,7 @@ class SerializerJsTranslator(declaration: KtPureClassOrObject,
         +JsVars(bitMasks.map { JsVars.JsVar(it.name, JsIntLiteral(0)) }, false)
 
         // var localProp0, localProp1, ...
-        val localProps = orderedProperties.map { JsNameRef(jsFun.scope.declareFreshName(it.name)) }
+        val localProps = orderedProperties.mapIndexed {i, _ -> JsNameRef(jsFun.scope.declareFreshName("local$i")) }
         +JsVars(localProps.map { JsVars.JsVar(it.name) }, true)
 
         //input = input.readBegin(...)
@@ -266,7 +266,7 @@ class SerializerJsTranslator(declaration: KtPureClassOrObject,
                         // input.readXxxElementValue
                         val sti = getSerialTypeInfo(property)
                         val innerSerial = serializerInstance(sti.serializer, property.module, property.type, property.genericIndex)
-                        val call = if (innerSerial == null) {
+                        val call: JsExpression = if (innerSerial == null) {
                             val unknownSer = (sti.elementMethodPrefix.isEmpty())
                             val readFunc =
                                     inputClass.getFuncDesc("read${sti.elementMethodPrefix}ElementValue")
@@ -278,13 +278,25 @@ class SerializerJsTranslator(declaration: KtPureClassOrObject,
                             JsInvocation(JsNameRef(readFunc, inputVar), readArgs)
                         }
                         else {
+                            val notSeenTest = propNotSeenTest(bitMasks[i / 32], i)
                             val readFunc =
                                     inputClass.getFuncDesc("read${sti.elementMethodPrefix}SerializableElementValue").single()
                                             .let { context.getNameForDescriptor(it) }
-                            JsInvocation(JsNameRef(readFunc, inputVar),
+                            val updateFunc =
-                                         serialClassDescRef,
+                                    inputClass.getFuncDesc("update${sti.elementMethodPrefix}SerializableElementValue").single()
-                                         JsIntLiteral(i),
+                                            .let { context.getNameForDescriptor(it) }
-                                         innerSerial)
+                            JsConditional(notSeenTest,
+                                          JsInvocation(JsNameRef(readFunc, inputVar),
+                                                       serialClassDescRef,
+                                                       JsIntLiteral(i),
+                                                       innerSerial),
+                                          JsInvocation(JsNameRef(updateFunc, inputVar),
+                                                       serialClassDescRef,
+                                                       JsIntLiteral(i),
+                                                       innerSerial,
+                                                       localProps[i]
+                                          )
+                            )
                         }
                         // localPropI = ...
                         +JsAstUtils.assignment(

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/backend/jvm/JVMCodegenUtil.kt

@@ -282,5 +282,6 @@ internal val org.jetbrains.org.objectweb.asm.Type.standardSerializer: String? ge
     "Ljava/util/HashMap;" -> "HashMapSerializer"
     "Ljava/util/Map\$Entry;" -> "MapEntrySerializer"
     "Lkotlin/Pair;" -> "PairSerializer"
+    "Lkotlin/Triple;" -> "TripleSerializer"
     else -> null
 }

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/backend/jvm/SerialInfoCodegenImpl.kt

@@ -76,12 +76,14 @@ class SerialInfoCodegenImpl(val codegen: ImplementationBodyCodegen, val thisClas
         codegen.generateMethod(constr, { _, _ ->
             load(0, thisAsmType)
             invokespecial("java/lang/Object", "<init>", "()V", false)
-            props.forEachIndexed { index, prop ->
+            var varOffset = 1
+            props.forEach { prop ->
                 val propType = codegen.typeMapper.mapType(prop.type)
                 val propFieldName = "_" + prop.name.identifier
                 load(0, thisAsmType)
-                load(index + 1, propType)
+                load(varOffset, propType)
                 putfield(thisAsmType.internalName, propFieldName, propType.descriptor)
+                varOffset += propType.size
             }
             areturn(Type.VOID_TYPE)
         })

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/backend/jvm/SerializerCodegenImpl.kt

@@ -280,12 +280,34 @@ class SerializerCodegenImpl(
                         aconst(codegen.typeMapper.mapType(property.type))
                         AsmUtil.wrapJavaClassIntoKClass(this)
                     }
-                    invokevirtual(kInputType.internalName,
+
-                                  "read" + sti.elementMethodPrefix + (if (useSerializer) "Serializable" else "") + "ElementValue",
+                    fun produceCall(update: Boolean) {
-                                  "(" + descType.descriptor + "I" +
+                        invokevirtual(kInputType.internalName,
-                                  (if (useSerializer) kSerialLoaderType.descriptor else "")
+                                      (if (update) "update" else "read") + sti.elementMethodPrefix + (if (useSerializer) "Serializable" else "") + "ElementValue",
-                                  + (if (unknownSer) AsmTypes.K_CLASS_TYPE.descriptor else "")
+                                      "(" + descType.descriptor + "I" +
-                                  + ")" + (if (sti.unit) "V" else sti.type.descriptor), false)
+                                      (if (useSerializer) kSerialLoaderType.descriptor else "")
+                                      + (if (unknownSer) AsmTypes.K_CLASS_TYPE.descriptor else "")
+                                      + (if (update) sti.type.descriptor else "")
+                                      + ")" + (if (sti.unit) "V" else sti.type.descriptor), false)
+                    }
+
+                    if (useSerializer) {
+                        // we can choose either it is read or update
+                        val readLabel = Label()
+                        val endL = Label()
+                        genValidateProperty(index, ::bitMaskOff)
+                        ificmpeq(readLabel)
+                        load(propVar, propertyType)
+                        produceCall(true)
+                        goTo(endL)
+                        visitLabel(readLabel)
+                        produceCall(false)
+                        visitLabel(endL)
+                    } else {
+                        // update not supported for primitive types
+                        produceCall(false)
+                    }
+
                     if (sti.unit) {
                         StackValue.putUnitInstance(this)
                     }

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/extensions/SerializationResolveExtension.kt

@@ -25,16 +25,14 @@ import org.jetbrains.kotlin.resolve.extensions.SyntheticResolveExtension
 import org.jetbrains.kotlin.resolve.lazy.LazyClassContext
 import org.jetbrains.kotlin.resolve.lazy.declarations.ClassMemberDeclarationProvider
 import org.jetbrains.kotlin.types.KotlinType
-import org.jetbrains.kotlinx.serialization.compiler.resolve.KSerializerDescriptorResolver
+import org.jetbrains.kotlinx.serialization.compiler.resolve.*
-import org.jetbrains.kotlinx.serialization.compiler.resolve.getSerializableClassDescriptorByCompanion
-import org.jetbrains.kotlinx.serialization.compiler.resolve.isInternalSerializable
-import org.jetbrains.kotlinx.serialization.compiler.resolve.serialInfoFqName
 import java.util.*
 
 class SerializationResolveExtension : SyntheticResolveExtension {
     override fun getSyntheticNestedClassNames(thisDescriptor: ClassDescriptor): List<Name> = when {
         thisDescriptor.annotations.hasAnnotation(serialInfoFqName) -> listOf(KSerializerDescriptorResolver.IMPL_NAME)
-        thisDescriptor.isInternalSerializable -> listOf(KSerializerDescriptorResolver.SERIALIZER_CLASS_NAME)
+        thisDescriptor.isInternalSerializable && !thisDescriptor.hasCompanionObjectAsSerializer ->
+            listOf(KSerializerDescriptorResolver.SERIALIZER_CLASS_NAME)
         else -> listOf()
     }
 
@@ -47,7 +45,8 @@ class SerializationResolveExtension : SyntheticResolveExtension {
     override fun generateSyntheticClasses(thisDescriptor: ClassDescriptor, name: Name, ctx: LazyClassContext, declarationProvider: ClassMemberDeclarationProvider, result: MutableSet<ClassDescriptor>) {
         if (thisDescriptor.annotations.hasAnnotation(serialInfoFqName) && name == KSerializerDescriptorResolver.IMPL_NAME)
             result.add(KSerializerDescriptorResolver.addSerialInfoImplClass(thisDescriptor, declarationProvider, ctx))
-        else if (thisDescriptor.isInternalSerializable && name == KSerializerDescriptorResolver.SERIALIZER_CLASS_NAME)
+        else if (thisDescriptor.isInternalSerializable && name == KSerializerDescriptorResolver.SERIALIZER_CLASS_NAME &&
+                 result.none { it.name == KSerializerDescriptorResolver.SERIALIZER_CLASS_NAME })
             result.add(KSerializerDescriptorResolver.addSerializerImplClass(thisDescriptor, declarationProvider, ctx))
         return
     }

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/resolve/KSerializationUtil.kt

@@ -26,6 +26,7 @@ import org.jetbrains.kotlin.resolve.descriptorUtil.module
 import org.jetbrains.kotlin.resolve.scopes.getDescriptorsFiltered
 import org.jetbrains.kotlin.types.*
 import org.jetbrains.kotlin.types.typeUtil.isSubtypeOf
+import org.jetbrains.kotlin.types.typeUtil.supertypes
 import org.jetbrains.kotlinx.serialization.compiler.resolve.KSerializerDescriptorResolver.SERIALIZER_CLASS_NAME
 
 internal val packageFqName = FqName("kotlinx.serialization")
@@ -122,15 +123,33 @@ internal val ClassDescriptor?.classSerializer: KotlinType?
         // serializer annotation on class?
         annotations.serializableWith?.let { return it }
         // default serializable?
-        if (isInternalSerializable) return this.unsubstitutedMemberScope
+        if (isInternalSerializable) {
-                .getDescriptorsFiltered(nameFilter = {it == SERIALIZER_CLASS_NAME})
+            // companion object serializer?
-                .filterIsInstance<ClassDescriptor>().singleOrNull()?.defaultType
+            if (hasCompanionObjectAsSerializer) return companionObjectDescriptor?.defaultType
+            // $serializer nested class
+            return this.unsubstitutedMemberScope
+                    .getDescriptorsFiltered(nameFilter = {it == SERIALIZER_CLASS_NAME})
+                    .filterIsInstance<ClassDescriptor>().singleOrNull()?.defaultType
+        }
         return null
     }
 
+internal val ClassDescriptor.hasCompanionObjectAsSerializer: Boolean
+    get() = companionObjectDescriptor?.annotations?.serializerForClass == this.defaultType
+
+internal fun checkSerializerNullability(classType: KotlinType, serializerType: KotlinType): KotlinType {
+    val castedToKSerial = requireNotNull(
+            serializerType.supertypes().find { isKSerializer(it) },
+            { "KSerializer is not a supertype of $serializerType" }
+    )
+    if (!classType.isMarkedNullable && castedToKSerial.arguments.first().type.isMarkedNullable)
+        throw IllegalStateException("Can't serialize non-nullable field of type ${classType} with nullable serializer ${serializerType}")
+    return serializerType
+}
+
 // serializer that was declared for this specific type or annotation from a class declaration
 val KotlinType.typeSerializer: KotlinType?
-    get() = this.annotations.serializableWith ?: (this.toClassDescriptor).classSerializer
+    get() = (this.annotations.serializableWith ?: (this.toClassDescriptor).classSerializer)?.let { checkSerializerNullability(this, it) }
 
 val KotlinType.genericIndex: Int?
     get() = (this.constructor.declarationDescriptor as? TypeParameterDescriptor)?.index

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/resolve/SerializableProperty.kt

@@ -24,7 +24,7 @@ class SerializableProperty(val descriptor: PropertyDescriptor, val isConstructor
     val type = descriptor.type
     val genericIndex = type.genericIndex
     val module = descriptor.module
-    val serializableWith = descriptor.annotations.serializableWith
+    val serializableWith = descriptor.annotations.serializableWith?.let { checkSerializerNullability(type, it) }
     val optional = descriptor.annotations.serialOptional
     val transient = descriptor.annotations.serialTransient
     val annotations = descriptor.annotations