[FIR] Unset EnhancedNullability on withNullability() calls

Note that `FlexibleNullability` doesn't
exist anymore: it was removed at
`65ea4e18`.

`preserveEnhancedNullability = true`
was needed because if we have an
`@EnhancedNullability DNN`, and we
substitute something into it, then
`withNullability` is called there becase
that's how DNNs work, not becase we want
to obtain some new type with a
different nullability.

This is backed by
the `compiler/testData/diagnostics/tests/j+k/integerNotNullable.kt` test
 where we first have
 `@EnhancedNullability T & Any`, then
 we substitute `{T -> kotlin/Int!}` and
 then inside
 `org.jetbrains.kotlin.fir.resolve.substitution.AbstractConeSubstitutor#substituteOriginal`
 (this is a DNN-specific function) we
 call `withNullability(NOT_NULL)`,
 and expect the attribute to be preserved. Otherwise this test would
 fail with
 `OVERLOAD_RESOLUTION_AMBIGUITY` for
 `IntBox().put(1)`.

^KT-50221 Fixed


Merge-request: KT-MR-11272
Merged-by: Nikolay Lunyak <Nikolay.Lunyak@jetbrains.com>

compiler/fir/analysis-tests/testData/resolve/inference/javaCollector.fir.txt

@@ -1,6 +1,6 @@
 FILE: javaCollector.kt
     public final fun foo(): R|kotlin/Unit| {
-        R|kotlin/collections/listOf|<R|kotlin/String|>(String()).R|SubstitutionOverride<kotlin/collections/List.stream: R|@EnhancedNullability java/util/stream/Stream<@EnhancedNullability kotlin/String>|>|().R|SubstitutionOverride<java/util/stream/Stream.collect: R|ft<R & Any, R?>|>|<R|ft<kotlin/collections/MutableMap<@EnhancedNullability kotlin/String!, kotlin/Int!>, kotlin/collections/Map<@EnhancedNullability kotlin/String!, kotlin/Int!>?>|, R|ft<CapturedType(*), CapturedType(*)?>|>(Q|java/util/stream/Collectors|.R|java/util/stream/Collectors.groupingBy*s|<R|@EnhancedNullability kotlin/String!|, R|@EnhancedNullability kotlin/String!|, R|ft<CapturedType(*), CapturedType(*)?>|, R|kotlin/Int!|>(groupingBy@fun <anonymous>(it: R|@EnhancedNullability kotlin/String!|): R|@EnhancedNullability kotlin/String!| <inline=NoInline>  {
+        R|kotlin/collections/listOf|<R|kotlin/String|>(String()).R|SubstitutionOverride<kotlin/collections/List.stream: R|@EnhancedNullability java/util/stream/Stream<@EnhancedNullability kotlin/String>|>|().R|SubstitutionOverride<java/util/stream/Stream.collect: R|ft<R & Any, R?>|>|<R|ft<kotlin/collections/MutableMap<kotlin/String!, kotlin/Int!>, kotlin/collections/Map<kotlin/String!, kotlin/Int!>?>|, R|ft<CapturedType(*), CapturedType(*)?>|>(Q|java/util/stream/Collectors|.R|java/util/stream/Collectors.groupingBy*s|<R|kotlin/String!|, R|kotlin/String!|, R|ft<CapturedType(*), CapturedType(*)?>|, R|kotlin/Int!|>(groupingBy@fun <anonymous>(it: R|@EnhancedNullability kotlin/String!|): R|@EnhancedNullability kotlin/String!| <inline=NoInline>  {
             ^ R|<local>/it|
         }
         , Q|java/util/stream/Collectors|.R|java/util/stream/Collectors.collectingAndThen*s|<R|kotlin/String!|, R|ft<CapturedType(*), CapturedType(*)?>|, R|kotlin/Long!|, R|kotlin/Int!|>(Q|java/util/stream/Collectors|.R|java/util/stream/Collectors.counting*s|<R|kotlin/String|>(), Q|kotlin/Long|::R|kotlin/Long.toInt|)))

compiler/fir/providers/src/org/jetbrains/kotlin/fir/resolve/substitution/Substitutors.kt

@@ -113,7 +113,8 @@ abstract class AbstractConeSubstitutor(protected val typeContext: ConeTypeContex
         val substituted = substitutedOriginal.withNullability(
             ConeNullability.NOT_NULL,
             typeContext,
-            substitutedOriginal.attributes.add(original.attributes)
+            substitutedOriginal.attributes.add(original.attributes),
+            preserveEnhancedNullability = true,
         )
         return ConeDefinitelyNotNullType.create(
             substituted, typeContext, avoidComprehensiveCheck = true,

compiler/fir/providers/src/org/jetbrains/kotlin/fir/types/TypeUtils.kt

@@ -34,6 +34,7 @@ import org.jetbrains.kotlin.fir.utils.exceptions.withConeTypeEntry
 import org.jetbrains.kotlin.resolve.calls.NewCommonSuperTypeCalculator
 import org.jetbrains.kotlin.types.*
 import org.jetbrains.kotlin.types.model.*
+import org.jetbrains.kotlin.utils.addToStdlib.butIf
 import org.jetbrains.kotlin.utils.addToStdlib.runIf
 import org.jetbrains.kotlin.utils.exceptions.errorWithAttachment
 import org.jetbrains.kotlin.fir.types.lowerBoundIfFlexible as coneLowerBoundIfFlexible
@@ -250,16 +251,21 @@ fun <T : ConeKotlinType> T.withNullability(
     nullability: ConeNullability,
     typeContext: ConeTypeContext,
     attributes: ConeAttributes = this.attributes,
+    preserveEnhancedNullability: Boolean = false,
 ): T {
-    if (this.nullability == nullability && this.attributes == attributes) {
+    val theAttributes = attributes.butIf(!preserveEnhancedNullability) {
+        it.remove(CompilerConeAttributes.EnhancedNullability)
+    }
+
+    if (this.nullability == nullability && this.attributes == theAttributes) {
         return this
     }
 
     @Suppress("UNCHECKED_CAST")
     return when (this) {
         is ConeErrorType -> this
-        is ConeClassLikeTypeImpl -> ConeClassLikeTypeImpl(lookupTag, typeArguments, nullability.isNullable, attributes)
+        is ConeClassLikeTypeImpl -> ConeClassLikeTypeImpl(lookupTag, typeArguments, nullability.isNullable, theAttributes)
-        is ConeTypeParameterTypeImpl -> ConeTypeParameterTypeImpl(lookupTag, nullability.isNullable, attributes)
+        is ConeTypeParameterTypeImpl -> ConeTypeParameterTypeImpl(lookupTag, nullability.isNullable, theAttributes)
         is ConeDynamicType -> this
         is ConeFlexibleType -> {
             if (nullability == ConeNullability.UNKNOWN) {
@@ -269,16 +275,16 @@ fun <T : ConeKotlinType> T.withNullability(
             }
             coneFlexibleOrSimpleType(
                 typeContext,
-                lowerBound.withNullability(nullability, typeContext),
+                lowerBound.withNullability(nullability, typeContext, preserveEnhancedNullability = preserveEnhancedNullability),
-                upperBound.withNullability(nullability, typeContext)
+                upperBound.withNullability(nullability, typeContext, preserveEnhancedNullability = preserveEnhancedNullability)
             )
         }
 
-        is ConeTypeVariableType -> ConeTypeVariableType(nullability, lookupTag, attributes)
+        is ConeTypeVariableType -> ConeTypeVariableType(nullability, lookupTag, theAttributes)
-        is ConeCapturedType -> ConeCapturedType(captureStatus, lowerType, nullability, constructor, attributes)
+        is ConeCapturedType -> ConeCapturedType(captureStatus, lowerType, nullability, constructor, theAttributes)
         is ConeIntersectionType -> when (nullability) {
             ConeNullability.NULLABLE -> this.mapTypes {
-                it.withNullability(nullability, typeContext)
+                it.withNullability(nullability, typeContext, preserveEnhancedNullability = preserveEnhancedNullability)
             }
 
             ConeNullability.UNKNOWN -> this // TODO: is that correct?
@@ -290,8 +296,12 @@ fun <T : ConeKotlinType> T.withNullability(
         is ConeStubTypeForTypeVariableInSubtyping -> ConeStubTypeForTypeVariableInSubtyping(constructor, nullability)
         is ConeDefinitelyNotNullType -> when (nullability) {
             ConeNullability.NOT_NULL -> this
-            ConeNullability.NULLABLE -> original.withNullability(nullability, typeContext)
+            ConeNullability.NULLABLE -> original.withNullability(
-            ConeNullability.UNKNOWN -> original.withNullability(nullability, typeContext)
+                nullability, typeContext, preserveEnhancedNullability = preserveEnhancedNullability,
+            )
+            ConeNullability.UNKNOWN -> original.withNullability(
+                nullability, typeContext, preserveEnhancedNullability = preserveEnhancedNullability,
+            )
         }
 
         is ConeIntegerLiteralConstantType -> ConeIntegerLiteralConstantTypeImpl(value, possibleTypes, isUnsigned, nullability)

compiler/testData/ir/irText/firProblems/AbstractMutableMap.fir.ir.txt

@@ -140,25 +140,25 @@ FILE fqName:<root> fileName:/AbstractMutableMap.kt
       $this: VALUE_PARAMETER name:<this> type:kotlin.collections.MutableMap<K of kotlin.collections.MutableMap, V of kotlin.collections.MutableMap>
       VALUE_PARAMETER name:p0 index:0 type:@[EnhancedNullability] K of <root>.MyMap
       VALUE_PARAMETER name:p1 index:1 type:@[EnhancedNullability] java.util.function.Function<in @[EnhancedNullability] K of <root>.MyMap, out @[EnhancedNullability] V of <root>.MyMap>
-    FUN FAKE_OVERRIDE name:computeIfPresent visibility:public modality:OPEN <> ($this:kotlin.collections.MutableMap<K of kotlin.collections.MutableMap, V of kotlin.collections.MutableMap>, p0:@[EnhancedNullability] K of <root>.MyMap, p1:@[EnhancedNullability] java.util.function.BiFunction<in @[EnhancedNullability] K of <root>.MyMap, in @[EnhancedNullability] V of <root>.MyMap, out V of <root>.MyMap?>) returnType:V of <root>.MyMap? [fake_override]
+    FUN FAKE_OVERRIDE name:computeIfPresent visibility:public modality:OPEN <> ($this:kotlin.collections.MutableMap<K of kotlin.collections.MutableMap, V of kotlin.collections.MutableMap>, p0:@[EnhancedNullability] K of <root>.MyMap, p1:@[EnhancedNullability] java.util.function.BiFunction<in @[EnhancedNullability] K of <root>.MyMap, in V of <root>.MyMap, out V of <root>.MyMap?>) returnType:V of <root>.MyMap? [fake_override]
       overridden:
         public open fun computeIfPresent (p0: @[EnhancedNullability] K of kotlin.collections.AbstractMutableMap, p1: @[EnhancedNullability] java.util.function.BiFunction<in @[EnhancedNullability] K of kotlin.collections.AbstractMutableMap, in @[EnhancedNullability] {V of kotlin.collections.AbstractMutableMap & Any}, out V of kotlin.collections.AbstractMutableMap?>): V of kotlin.collections.AbstractMutableMap? declared in kotlin.collections.AbstractMutableMap
       $this: VALUE_PARAMETER name:<this> type:kotlin.collections.MutableMap<K of kotlin.collections.MutableMap, V of kotlin.collections.MutableMap>
       VALUE_PARAMETER name:p0 index:0 type:@[EnhancedNullability] K of <root>.MyMap
-      VALUE_PARAMETER name:p1 index:1 type:@[EnhancedNullability] java.util.function.BiFunction<in @[EnhancedNullability] K of <root>.MyMap, in @[EnhancedNullability] V of <root>.MyMap, out V of <root>.MyMap?>
+      VALUE_PARAMETER name:p1 index:1 type:@[EnhancedNullability] java.util.function.BiFunction<in @[EnhancedNullability] K of <root>.MyMap, in V of <root>.MyMap, out V of <root>.MyMap?>
     FUN FAKE_OVERRIDE name:compute visibility:public modality:OPEN <> ($this:kotlin.collections.MutableMap<K of kotlin.collections.MutableMap, V of kotlin.collections.MutableMap>, p0:@[EnhancedNullability] K of <root>.MyMap, p1:@[EnhancedNullability] java.util.function.BiFunction<in @[EnhancedNullability] K of <root>.MyMap, in V of <root>.MyMap?, out V of <root>.MyMap?>) returnType:V of <root>.MyMap? [fake_override]
       overridden:
         public open fun compute (p0: @[EnhancedNullability] K of kotlin.collections.AbstractMutableMap, p1: @[EnhancedNullability] java.util.function.BiFunction<in @[EnhancedNullability] K of kotlin.collections.AbstractMutableMap, in V of kotlin.collections.AbstractMutableMap?, out V of kotlin.collections.AbstractMutableMap?>): V of kotlin.collections.AbstractMutableMap? declared in kotlin.collections.AbstractMutableMap
       $this: VALUE_PARAMETER name:<this> type:kotlin.collections.MutableMap<K of kotlin.collections.MutableMap, V of kotlin.collections.MutableMap>
       VALUE_PARAMETER name:p0 index:0 type:@[EnhancedNullability] K of <root>.MyMap
       VALUE_PARAMETER name:p1 index:1 type:@[EnhancedNullability] java.util.function.BiFunction<in @[EnhancedNullability] K of <root>.MyMap, in V of <root>.MyMap?, out V of <root>.MyMap?>
-    FUN FAKE_OVERRIDE name:merge visibility:public modality:OPEN <> ($this:kotlin.collections.MutableMap<K of kotlin.collections.MutableMap, V of kotlin.collections.MutableMap>, p0:@[EnhancedNullability] K of <root>.MyMap, p1:@[EnhancedNullability] {V of <root>.MyMap & Any}, p2:@[EnhancedNullability] java.util.function.BiFunction<in @[EnhancedNullability] V of <root>.MyMap, in @[EnhancedNullability] V of <root>.MyMap, out V of <root>.MyMap?>) returnType:V of <root>.MyMap? [fake_override]
+    FUN FAKE_OVERRIDE name:merge visibility:public modality:OPEN <> ($this:kotlin.collections.MutableMap<K of kotlin.collections.MutableMap, V of kotlin.collections.MutableMap>, p0:@[EnhancedNullability] K of <root>.MyMap, p1:@[EnhancedNullability] {V of <root>.MyMap & Any}, p2:@[EnhancedNullability] java.util.function.BiFunction<in V of <root>.MyMap, in V of <root>.MyMap, out V of <root>.MyMap?>) returnType:V of <root>.MyMap? [fake_override]
       overridden:
         public open fun merge (p0: @[EnhancedNullability] K of kotlin.collections.AbstractMutableMap, p1: @[EnhancedNullability] {V of kotlin.collections.AbstractMutableMap & Any}, p2: @[EnhancedNullability] java.util.function.BiFunction<in @[EnhancedNullability] {V of kotlin.collections.AbstractMutableMap & Any}, in @[EnhancedNullability] {V of kotlin.collections.AbstractMutableMap & Any}, out V of kotlin.collections.AbstractMutableMap?>): V of kotlin.collections.AbstractMutableMap? declared in kotlin.collections.AbstractMutableMap
       $this: VALUE_PARAMETER name:<this> type:kotlin.collections.MutableMap<K of kotlin.collections.MutableMap, V of kotlin.collections.MutableMap>
       VALUE_PARAMETER name:p0 index:0 type:@[EnhancedNullability] K of <root>.MyMap
       VALUE_PARAMETER name:p1 index:1 type:@[EnhancedNullability] {V of <root>.MyMap & Any}
-      VALUE_PARAMETER name:p2 index:2 type:@[EnhancedNullability] java.util.function.BiFunction<in @[EnhancedNullability] V of <root>.MyMap, in @[EnhancedNullability] V of <root>.MyMap, out V of <root>.MyMap?>
+      VALUE_PARAMETER name:p2 index:2 type:@[EnhancedNullability] java.util.function.BiFunction<in V of <root>.MyMap, in V of <root>.MyMap, out V of <root>.MyMap?>
     FUN FAKE_OVERRIDE name:equals visibility:public modality:OPEN <> ($this:java.util.AbstractMap<K of java.util.AbstractMap, V of java.util.AbstractMap>, p0:kotlin.Any?) returnType:kotlin.Boolean [fake_override,operator]
       overridden:
         public open fun equals (p0: kotlin.Any?): kotlin.Boolean declared in kotlin.collections.AbstractMutableMap