[FIR] Ignore CFA-only edges in DFA even if they are dead

^KT-53920 Fixed

analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/diagnostic/compiler/based/DiagnosisCompilerFirTestdataTestGenerated.java

@@ -1075,6 +1075,12 @@ public class DiagnosisCompilerFirTestdataTestGenerated extends AbstractDiagnosis
                 runTest("compiler/fir/analysis-tests/testData/resolve/cfg/annotatedLocalClass.kt");
             }
 
+            @Test
+            @TestMetadata("anonymousObjectInUnreachableCode.kt")
+            public void testAnonymousObjectInUnreachableCode() throws Exception {
+                runTest("compiler/fir/analysis-tests/testData/resolve/cfg/anonymousObjectInUnreachableCode.kt");
+            }
+
             @Test
             @TestMetadata("binaryOperations.kt")
             public void testBinaryOperations() throws Exception {

compiler/fir/analysis-tests/legacy-fir-tests/tests-gen/org/jetbrains/kotlin/fir/LazyBodyIsNotTouchedTilContractsPhaseTestGenerated.java

@@ -931,6 +931,11 @@ public class LazyBodyIsNotTouchedTilContractsPhaseTestGenerated extends Abstract
             runTest("compiler/fir/analysis-tests/testData/resolve/cfg/annotatedLocalClass.kt");
         }
 
+        @TestMetadata("anonymousObjectInUnreachableCode.kt")
+        public void testAnonymousObjectInUnreachableCode() throws Exception {
+            runTest("compiler/fir/analysis-tests/testData/resolve/cfg/anonymousObjectInUnreachableCode.kt");
+        }
+
         @TestMetadata("binaryOperations.kt")
         public void testBinaryOperations() throws Exception {
             runTest("compiler/fir/analysis-tests/testData/resolve/cfg/binaryOperations.kt");

compiler/fir/analysis-tests/testData/resolve/cfg/anonymousObjectInUnreachableCode.fir.txt

@@ -0,0 +1,11 @@
+FILE: anonymousObjectInUnreachableCode.kt
+    public final fun test(): R|kotlin/Unit| {
+        ^test Unit
+        object : R|kotlin/Any| {
+            private constructor(): R|<anonymous>| {
+                super<R|kotlin/Any|>()
+            }
+
+        }
+
+    }

compiler/fir/analysis-tests/testData/resolve/cfg/anonymousObjectInUnreachableCode.kt

@@ -0,0 +1,6 @@
+// ISSUE: KT-53920
+
+fun test() {
+    return
+    object {}
+}

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirDiagnosticTestGenerated.java

@@ -1075,6 +1075,12 @@ public class FirDiagnosticTestGenerated extends AbstractFirDiagnosticTest {
                 runTest("compiler/fir/analysis-tests/testData/resolve/cfg/annotatedLocalClass.kt");
             }
 
+            @Test
+            @TestMetadata("anonymousObjectInUnreachableCode.kt")
+            public void testAnonymousObjectInUnreachableCode() throws Exception {
+                runTest("compiler/fir/analysis-tests/testData/resolve/cfg/anonymousObjectInUnreachableCode.kt");
+            }
+
             @Test
             @TestMetadata("binaryOperations.kt")
             public void testBinaryOperations() throws Exception {

compiler/fir/analysis-tests/tests-gen/org/jetbrains/kotlin/test/runners/FirDiagnosticsWithLightTreeTestGenerated.java

@@ -1075,6 +1075,12 @@ public class FirDiagnosticsWithLightTreeTestGenerated extends AbstractFirDiagnos
                 runTest("compiler/fir/analysis-tests/testData/resolve/cfg/annotatedLocalClass.kt");
             }
 
+            @Test
+            @TestMetadata("anonymousObjectInUnreachableCode.kt")
+            public void testAnonymousObjectInUnreachableCode() throws Exception {
+                runTest("compiler/fir/analysis-tests/testData/resolve/cfg/anonymousObjectInUnreachableCode.kt");
+            }
+
             @Test
             @TestMetadata("binaryOperations.kt")
             public void testBinaryOperations() throws Exception {

compiler/fir/resolve/src/org/jetbrains/kotlin/fir/resolve/dfa/FirDataFlowAnalyzer.kt

@@ -355,7 +355,6 @@ abstract class FirDataFlowAnalyzer<FLOW : Flow>(
     }
 
     fun exitAnonymousObject(anonymousObject: FirAnonymousObject): ControlFlowGraph {
-        // TODO: support capturing of mutable properties, KT-44877
         val (node, controlFlowGraph) = graphBuilder.exitAnonymousObject(anonymousObject)
         node.mergeIncomingFlow()
         return controlFlowGraph
@@ -1555,13 +1554,13 @@ abstract class FirDataFlowAnalyzer<FLOW : Flow>(
         var deadForwardCount = 0
         for (previousNode in previousNodes) {
             val incomingEdgeKind = incomingEdges.getValue(previousNode).kind
-            if (isDead) {
-                if (!incomingEdgeKind.isBack) {
-                    previousFlows += previousNode.flow
-                }
+
+            if (isDead && incomingEdgeKind.usedInDeadDfa) {
+                previousFlows += previousNode.flow
             } else if (incomingEdgeKind.usedInDfa) {
                 previousFlows += previousNode.flow
             }
+
             if (incomingEdgeKind == EdgeKind.DeadForward) {
                 deadForwardCount++
             }

compiler/fir/semantics/src/org/jetbrains/kotlin/fir/resolve/dfa/cfg/ControlFlowGraph.kt

@@ -157,17 +157,18 @@ class ReturnPath(
 ) : EdgeLabel(label = "\"return@${returnTargetSymbol.callableId}\"")
 
 enum class EdgeKind(
-    val usedInDfa: Boolean,
+    val usedInDfa: Boolean, // propagate flow to alive nodes
+    val usedInDeadDfa: Boolean, // propagate flow to dead nodes
     val usedInCfa: Boolean,
     val isBack: Boolean,
     val isDead: Boolean
 ) {
-    Forward(usedInDfa = true, usedInCfa = true, isBack = false, isDead = false),
-    DeadForward(usedInDfa = false, usedInCfa = true, isBack = false, isDead = true),
-    DfgForward(usedInDfa = true, usedInCfa = false, isBack = false, isDead = false),
-    CfgForward(usedInDfa = false, usedInCfa = true, isBack = false, isDead = false),
-    CfgBackward(usedInDfa = false, usedInCfa = true, isBack = true, isDead = false),
-    DeadBackward(usedInDfa = false, usedInCfa = true, isBack = true, isDead = true)
+    Forward(usedInDfa = true, usedInDeadDfa = true, usedInCfa = true, isBack = false, isDead = false),
+    DeadForward(usedInDfa = false, usedInDeadDfa = true, usedInCfa = true, isBack = false, isDead = true),
+    DfgForward(usedInDfa = true, usedInDeadDfa = true, usedInCfa = false, isBack = false, isDead = false),
+    CfgForward(usedInDfa = false, usedInDeadDfa = false, usedInCfa = true, isBack = false, isDead = false),
+    CfgBackward(usedInDfa = false, usedInDeadDfa = false, usedInCfa = true, isBack = true, isDead = false),
+    DeadBackward(usedInDfa = false, usedInDeadDfa = false, usedInCfa = true, isBack = true, isDead = true)
 }
 
 @OptIn(ExperimentalStdlibApi::class)