[FIR serializer] Support type approximation

Before this commit we relied on a fact that exotic types aren't
possible at serialization stage. However, in non-ABI mode we can get
even a private function (e.g. top-level private), so we have to perform
this approximation.

compiler/fir/fir-serialization/src/org/jetbrains/kotlin/fir/serialization/FirElementSerializer.kt

@@ -48,6 +48,8 @@ import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.RequireKotlinConstants
 import org.jetbrains.kotlin.serialization.deserialization.ProtoEnumFlags
+import org.jetbrains.kotlin.types.AbstractTypeApproximator
+import org.jetbrains.kotlin.types.TypeApproximatorConfiguration
 
 class FirElementSerializer private constructor(
     private val session: FirSession,
@@ -57,6 +59,7 @@ class FirElementSerializer private constructor(
     private val typeTable: MutableTypeTable,
     private val versionRequirementTable: MutableVersionRequirementTable?,
     private val serializeTypeTableToFunction: Boolean,
+    private val typeApproximator: AbstractTypeApproximator,
 ) {
     private val contractSerializer = FirContractSerializer()
 
@@ -261,7 +264,7 @@ class FirElementSerializer private constructor(
         if (useTypeTable()) {
             builder.returnTypeId = local.typeId(property.returnTypeRef)
         } else {
-            builder.setReturnType(local.typeProto(property.returnTypeRef))
+            builder.setReturnType(local.typeProto(property.returnTypeRef, toSuper = true))
         }
 
         for (typeParameter in property.typeParameters) {
@@ -334,7 +337,7 @@ class FirElementSerializer private constructor(
         if (useTypeTable()) {
             builder.returnTypeId = local.typeId(function.returnTypeRef)
         } else {
-            builder.setReturnType(local.typeProto(function.returnTypeRef))
+            builder.setReturnType(local.typeProto(function.returnTypeRef, toSuper = true))
         }
 
         for (typeParameter in function.typeParameters) {
@@ -546,8 +549,16 @@ class FirElementSerializer private constructor(
 
     fun typeId(type: ConeKotlinType): Int = typeTable[typeProto(type)]
 
-    private fun typeProto(typeRef: FirTypeRef): ProtoBuf.Type.Builder {
+    private fun typeProto(typeRef: FirTypeRef, toSuper: Boolean = false): ProtoBuf.Type.Builder {
-        return typeProto(typeRef.coneType).also {
+        val approximatedType = when (typeRef.coneType) {
+            is ConeIntegerLiteralType,
+            is ConeCapturedType,
+            is ConeDefinitelyNotNullType,
+            is ConeIntersectionType ->
+                typeRef.approximated(typeApproximator, toSuper)
+            else -> typeRef
+        }
+        return typeProto(approximatedType.coneType).also {
             extension.serializeType(typeRef, it)
         }
     }
@@ -588,6 +599,16 @@ class FirElementSerializer private constructor(
                     builder.typeParameter = getTypeParameterId(typeParameter)
                 }
             }
+            is ConeCapturedType,
+            is ConeDefinitelyNotNullType,
+            is ConeIntersectionType,
+            is ConeIntegerLiteralType -> {
+                val approximatedType = typeApproximator.approximateToSubType(type, TypeApproximatorConfiguration.PublicDeclaration)
+                assert(approximatedType != type && approximatedType is ConeKotlinType) {
+                    "Approximation failed: ${type.render()}"
+                }
+                return typeProto(approximatedType as ConeKotlinType)
+            }
             else -> {
                 throw AssertionError("Should not be here: ${type::class.java}")
             }
@@ -690,7 +711,8 @@ class FirElementSerializer private constructor(
     private fun createChildSerializer(declaration: FirDeclaration): FirElementSerializer =
         FirElementSerializer(
             session, declaration, Interner(typeParameters), extension,
-            typeTable, versionRequirementTable, serializeTypeTableToFunction = false
+            typeTable, versionRequirementTable, serializeTypeTableToFunction = false,
+            typeApproximator
         )
 
     val stringTable: FirElementAwareStringTable
@@ -816,33 +838,44 @@ class FirElementSerializer private constructor(
 
     companion object {
         @JvmStatic
-        fun createTopLevel(session: FirSession, extension: FirSerializerExtension): FirElementSerializer =
+        fun createTopLevel(
+            session: FirSession,
+            extension: FirSerializerExtension,
+            typeApproximator: AbstractTypeApproximator,
+        ): FirElementSerializer =
             FirElementSerializer(
                 session, null,
                 Interner(), extension, MutableTypeTable(), MutableVersionRequirementTable(),
-                serializeTypeTableToFunction = false
+                serializeTypeTableToFunction = false,
+                typeApproximator
             )
 
         @JvmStatic
-        fun createForLambda(session: FirSession, extension: FirSerializerExtension): FirElementSerializer =
+        fun createForLambda(
+            session: FirSession,
+            extension: FirSerializerExtension,
+            typeApproximator: AbstractTypeApproximator,
+        ): FirElementSerializer =
             FirElementSerializer(
                 session, null,
                 Interner(), extension, MutableTypeTable(),
-                versionRequirementTable = null, serializeTypeTableToFunction = true
+                versionRequirementTable = null, serializeTypeTableToFunction = true,
+                typeApproximator
             )
 
         @JvmStatic
         fun create(
             klass: FirClass<*>,
             extension: FirSerializerExtension,
-            parentSerializer: FirElementSerializer?
+            parentSerializer: FirElementSerializer?,
+            typeApproximator: AbstractTypeApproximator,
         ): FirElementSerializer {
             val parentClassId = klass.symbol.classId.outerClassId
             val parent = if (parentClassId != null && !parentClassId.isLocal) {
                 val parentClass = klass.session.firSymbolProvider.getClassLikeSymbolByFqName(parentClassId)!!.fir as FirRegularClass
-                parentSerializer ?: create(parentClass, extension, null)
+                parentSerializer ?: create(parentClass, extension, null, typeApproximator)
             } else {
-                createTopLevel(klass.session, extension)
+                createTopLevel(klass.session, extension, typeApproximator)
             }
 
             // Calculate type parameter ids for the outer class beforehand, as it would've had happened if we were always
@@ -859,7 +892,8 @@ class FirElementSerializer private constructor(
                 } else {
                     MutableVersionRequirementTable()
                 },
-                serializeTypeTableToFunction = false
+                serializeTypeTableToFunction = false,
+                typeApproximator
             )
             for (typeParameter in klass.typeParameters) {
                 if (typeParameter !is FirTypeParameter) continue

compiler/fir/fir2ir/jvm-backend/src/org/jetbrains/kotlin/fir/backend/jvm/FirMetadataSerializer.kt

@@ -15,6 +15,7 @@ import org.jetbrains.kotlin.fir.declarations.FirDeclarationOrigin
 import org.jetbrains.kotlin.fir.declarations.FirFunction
 import org.jetbrains.kotlin.fir.declarations.FirTypeParameter
 import org.jetbrains.kotlin.fir.declarations.builder.*
+import org.jetbrains.kotlin.fir.diagnostics.ConeIntermediateDiagnostic
 import org.jetbrains.kotlin.fir.typeContext
 import org.jetbrains.kotlin.fir.scopes.impl.withReplacedConeType
 import org.jetbrains.kotlin.fir.serialization.FirElementSerializer
@@ -27,6 +28,7 @@ import org.jetbrains.kotlin.metadata.jvm.serialization.JvmStringTable
 import org.jetbrains.kotlin.protobuf.MessageLite
 import org.jetbrains.kotlin.types.AbstractTypeApproximator
 import org.jetbrains.kotlin.types.TypeApproximatorConfiguration
+import org.jetbrains.kotlin.types.model.SimpleTypeMarker
 import org.jetbrains.org.objectweb.asm.Type
 import org.jetbrains.org.objectweb.asm.commons.Method
 
@@ -38,7 +40,11 @@ class FirMetadataSerializer(
     private val serializationBindings: JvmSerializationBindings,
     parent: MetadataSerializer?
 ) : MetadataSerializer {
-    private val approximator = object : AbstractTypeApproximator(session.typeContext) {}
+    private val approximator = object : AbstractTypeApproximator(session.typeContext) {
+        override fun createErrorType(debugName: String): SimpleTypeMarker {
+            return ConeKotlinErrorType(ConeIntermediateDiagnostic(debugName))
+        }
+    }
 
     private fun FirTypeRef.approximated(toSuper: Boolean): FirTypeRef {
         val approximatedType = if (toSuper)
@@ -72,10 +78,10 @@ class FirMetadataSerializer(
     private val serializer: FirElementSerializer? =
         when (val metadata = irClass.metadata) {
             is FirMetadataSource.Class -> FirElementSerializer.create(
-                metadata.fir, serializerExtension, (parent as? FirMetadataSerializer)?.serializer
+                metadata.fir, serializerExtension, (parent as? FirMetadataSerializer)?.serializer, approximator
             )
-            is FirMetadataSource.File -> FirElementSerializer.createTopLevel(metadata.session, serializerExtension)
+            is FirMetadataSource.File -> FirElementSerializer.createTopLevel(metadata.session, serializerExtension, approximator)
-            is FirMetadataSource.Function -> FirElementSerializer.createForLambda(metadata.session, serializerExtension)
+            is FirMetadataSource.Function -> FirElementSerializer.createForLambda(metadata.session, serializerExtension, approximator)
             else -> null
         }
 

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/types/TypeUtils.kt

@@ -7,10 +7,13 @@ package org.jetbrains.kotlin.fir.types
 
 import org.jetbrains.kotlin.fir.FirSession
 import org.jetbrains.kotlin.fir.resolve.fullyExpandedType
+import org.jetbrains.kotlin.fir.scopes.impl.withReplacedConeType
 import org.jetbrains.kotlin.fir.types.impl.ConeClassLikeTypeImpl
 import org.jetbrains.kotlin.fir.types.impl.ConeTypeParameterTypeImpl
 import org.jetbrains.kotlin.resolve.calls.NewCommonSuperTypeCalculator
 import org.jetbrains.kotlin.types.AbstractStrictEqualityTypeChecker
+import org.jetbrains.kotlin.types.AbstractTypeApproximator
+import org.jetbrains.kotlin.types.TypeApproximatorConfiguration
 
 fun ConeInferenceContext.commonSuperTypeOrNull(types: List<ConeKotlinType>): ConeKotlinType? {
     return when (types.size) {
@@ -171,3 +174,15 @@ fun FirTypeRef.isUnsafeVarianceType(session: FirSession): Boolean {
     return coneTypeSafe<ConeKotlinType>()?.isUnsafeVarianceType(session) == true
 }
 
+fun FirTypeRef.approximated(
+    typeApproximator: AbstractTypeApproximator,
+    toSuper: Boolean,
+    conf: TypeApproximatorConfiguration = TypeApproximatorConfiguration.PublicDeclaration
+): FirTypeRef {
+    val approximatedType = if (toSuper)
+        typeApproximator.approximateToSuperType(coneType, conf)
+    else
+        typeApproximator.approximateToSubType(coneType, conf)
+    return withReplacedConeType(approximatedType as? ConeKotlinType)
+}
+