Refactor ConeOverloadConflictResolver.chooseMaximallySpecificCandidates

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/calls/ConeOverloadConflictResolver.kt

@@ -88,14 +88,16 @@ class ConeOverloadConflictResolver(
         )
         return chooseMaximallySpecificCandidates(
             candidatesWithoutOverrides,
-            // (in compatibility mode the next two are already filtered on tower resolver level)
+            DiscriminationFlags(
-            discriminateLowPrioritySAMs = noCompatibilityMode,
+                // (in compatibility mode the next two are already filtered on tower resolver level)
-            discriminateAdaptationsInPostponedAtoms = noCompatibilityMode,
+                lowPrioritySAMs = noCompatibilityMode,
-            discriminateGenerics,
+                adaptationsInPostponedAtoms = noCompatibilityMode,
-            discriminateAbstracts,
+                generics = discriminateGenerics,
-            discriminateSAMs = true,
+                abstracts = discriminateAbstracts,
-            discriminateSuspendConversions = true,
+                SAMs = true,
-            discriminateByUnwrappedSmartCastOrigin = true,
+                suspendConversions = true,
+                byUnwrappedSmartCastOrigin = true,
+            )
         )
     }
 
@@ -162,119 +164,67 @@ class ConeOverloadConflictResolver(
         return candidates.filterTo(mutableSetOf()) { it.callInfo.candidateForCommonInvokeReceiver == bestInvokeReceiver }
     }
 
+    private data class DiscriminationFlags(
+        val lowPrioritySAMs: Boolean,
+        val adaptationsInPostponedAtoms: Boolean,
+        val generics: Boolean,
+        val abstracts: Boolean,
+        val SAMs: Boolean,
+        val suspendConversions: Boolean,
+        val byUnwrappedSmartCastOrigin: Boolean,
+    )
+
     private fun chooseMaximallySpecificCandidates(
         candidates: Set<Candidate>,
-        discriminateLowPrioritySAMs: Boolean,
+        discriminationFlags: DiscriminationFlags
-        discriminateAdaptationsInPostponedAtoms: Boolean,
-        discriminateGenerics: Boolean,
-        discriminateAbstracts: Boolean,
-        // Only set to 'false' by recursive calls when the relevant discrimination kind has been already applied
-        discriminateSAMs: Boolean,
-        discriminateSuspendConversions: Boolean,
-        discriminateByUnwrappedSmartCastOrigin: Boolean,
     ): Set<Candidate> {
-        if (discriminateLowPrioritySAMs) {
+        if (discriminationFlags.lowPrioritySAMs) {
-            val filtered = candidates.filterTo(mutableSetOf()) { !it.shouldHaveLowPriorityDueToSAM(transformerComponents) }
+            filterCandidatesByDiscriminationFlag(
-            when (filtered.size) {
+                candidates,
-                1 -> return filtered
+                { !it.shouldHaveLowPriorityDueToSAM(transformerComponents) },
-                0, candidates.size -> {
+                { discriminationFlags.copy(lowPrioritySAMs = false) },
-                }
+            )?.let { return it }
-                else -> return chooseMaximallySpecificCandidates(
-                    filtered,
-                    discriminateLowPrioritySAMs = false,
-                    discriminateAdaptationsInPostponedAtoms,
-                    discriminateGenerics,
-                    discriminateAbstracts,
-                    discriminateSAMs,
-                    discriminateSuspendConversions,
-                    discriminateByUnwrappedSmartCastOrigin,
-                )
-            }
         }
 
-        if (discriminateAdaptationsInPostponedAtoms) {
+        if (discriminationFlags.adaptationsInPostponedAtoms) {
-            val filtered = candidates.filterTo(mutableSetOf()) { !it.hasPostponedAtomWithAdaptation() }
+            filterCandidatesByDiscriminationFlag(
-            when (filtered.size) {
+                candidates,
-                1 -> return filtered
+                { !it.hasPostponedAtomWithAdaptation() },
-                0, candidates.size -> {
+                { discriminationFlags.copy(adaptationsInPostponedAtoms = false) },
-                }
+            )?.let { return it }
-                else -> return chooseMaximallySpecificCandidates(
-                    filtered,
-                    discriminateLowPrioritySAMs,
-                    discriminateAdaptationsInPostponedAtoms = false,
-                    discriminateGenerics,
-                    discriminateAbstracts,
-                    discriminateSAMs,
-                    discriminateSuspendConversions,
-                    discriminateByUnwrappedSmartCastOrigin,
-                )
-            }
         }
 
         findMaximallySpecificCall(candidates, false)?.let { return setOf(it) }
 
-        if (discriminateGenerics) {
+        if (discriminationFlags.generics) {
             findMaximallySpecificCall(candidates, true)?.let { return setOf(it) }
         }
 
-        if (discriminateSAMs) {
+        if (discriminationFlags.SAMs) {
-            val filtered = candidates.filterTo(mutableSetOf()) { !it.usesSAM }
+            filterCandidatesByDiscriminationFlag(
-            when (filtered.size) {
+                candidates,
-                1 -> return filtered
+                { !it.usesSAM },
-                0, candidates.size -> {
+                { discriminationFlags.copy(SAMs = false) },
-                }
+            )?.let { return it }
-                else -> return chooseMaximallySpecificCandidates(
-                    filtered,
-                    discriminateLowPrioritySAMs,
-                    discriminateAdaptationsInPostponedAtoms,
-                    discriminateGenerics,
-                    discriminateAbstracts,
-                    discriminateSAMs = false,
-                    discriminateSuspendConversions,
-                    discriminateByUnwrappedSmartCastOrigin,
-                )
-            }
         }
 
-        if (discriminateSuspendConversions) {
+        if (discriminationFlags.suspendConversions) {
-            val filtered = candidates.filterTo(mutableSetOf()) { !it.usesFunctionConversion }
+            filterCandidatesByDiscriminationFlag(
-            when (filtered.size) {
+                candidates,
-                1 -> return filtered
+                { !it.usesFunctionConversion },
-                0, candidates.size -> {
+                { discriminationFlags.copy(suspendConversions = false) },
-                }
+            )?.let { return it }
-                else -> return chooseMaximallySpecificCandidates(
-                    filtered,
-                    discriminateLowPrioritySAMs,
-                    discriminateAdaptationsInPostponedAtoms,
-                    discriminateGenerics,
-                    discriminateAbstracts,
-                    discriminateSAMs,
-                    discriminateSuspendConversions = false,
-                    discriminateByUnwrappedSmartCastOrigin,
-                )
-            }
         }
 
-        if (discriminateAbstracts) {
+        if (discriminationFlags.abstracts) {
-            val filtered = candidates.filterTo(mutableSetOf()) { (it.symbol.fir as? FirMemberDeclaration)?.modality != Modality.ABSTRACT }
+            filterCandidatesByDiscriminationFlag(
-            when (filtered.size) {
+                candidates,
-                1 -> return filtered
+                { (it.symbol.fir as? FirMemberDeclaration)?.modality != Modality.ABSTRACT },
-                0, candidates.size -> {
+                { discriminationFlags.copy(abstracts = false) },
-                }
+            )?.let { return it }
-                else -> return chooseMaximallySpecificCandidates(
-                    filtered,
-                    discriminateLowPrioritySAMs,
-                    discriminateAdaptationsInPostponedAtoms,
-                    discriminateGenerics,
-                    discriminateAbstracts = false,
-                    discriminateSAMs,
-                    discriminateSuspendConversions,
-                    discriminateByUnwrappedSmartCastOrigin,
-                )
-            }
         }
 
-        if (discriminateByUnwrappedSmartCastOrigin) {
+        if (discriminationFlags.byUnwrappedSmartCastOrigin) {
             // In case of MemberScopeTowerLevel with smart cast dispatch receiver, we may create candidates both from smart cast type and
             // from the member scope of original expression's type (without smart cast).
             // It might be necessary because the ones from smart cast might be invisible (e.g., because they are protected in other class).
@@ -302,22 +252,11 @@ class ConeOverloadConflictResolver(
             // See more details at KT-51460, KT-55722, KT-56310 and relevant tests
             //    testData/diagnostics/tests/visibility/moreSpecificProtectedSimple.kt
             //    testData/diagnostics/tests/smartCasts/kt51460.kt
-            val filtered = candidates.filterTo(mutableSetOf()) { !it.isFromOriginalTypeInPresenceOfSmartCast }
+            filterCandidatesByDiscriminationFlag(
-            when (filtered.size) {
+                candidates,
-                1 -> return filtered
+                { !it.isFromOriginalTypeInPresenceOfSmartCast },
-                0, candidates.size -> {
+                { discriminationFlags.copy(byUnwrappedSmartCastOrigin = false) },
-                }
+            )?.let { return it }
-                else -> return chooseMaximallySpecificCandidates(
-                    filtered,
-                    discriminateLowPrioritySAMs,
-                    discriminateAdaptationsInPostponedAtoms,
-                    discriminateGenerics,
-                    discriminateAbstracts,
-                    discriminateSAMs,
-                    discriminateSuspendConversions,
-                    discriminateByUnwrappedSmartCastOrigin = false,
-                )
-            }
         }
 
         val filtered = candidates.filterTo(mutableSetOf()) { it.usesSAM }
@@ -328,6 +267,19 @@ class ConeOverloadConflictResolver(
         return candidates
     }
 
+    private inline fun filterCandidatesByDiscriminationFlag(
+        candidates: Set<Candidate>,
+        filter: (Candidate) -> Boolean,
+        newFlags: () -> DiscriminationFlags,
+    ): Set<Candidate>? {
+        val filtered = candidates.filterTo(mutableSetOf()) { filter(it) }
+        return when (filtered.size) {
+            1 -> filtered
+            0, candidates.size -> null
+            else -> chooseMaximallySpecificCandidates(filtered, newFlags())
+        }
+    }
+
     private fun Candidate.hasPostponedAtomWithAdaptation(): Boolean {
         return postponedAtoms.any {
             it is ResolvedCallableReferenceAtom &&