jvm-abi-gen: only keep relevant InnerClasses from current module

If a class is not referenced by the ABI class file, its InnerClasses
entry should only be kept if it's the current class or a descendant
of the current class.

plugins/jvm-abi-gen/src/org/jetbrains/kotlin/jvm/abi/JvmAbiOutputExtension.kt

@@ -73,7 +73,7 @@ class JvmAbiOutputExtension(
                     else -> /* abiInfo is AbiClassInfo.Stripped */ {
                         val methodInfo = (abiInfo as AbiClassInfo.Stripped).methodInfo
                         val innerClassInfos = mutableMapOf<String, InnerClassInfo>()
-                        val innerClassesToKeep = abiClassInfos.keys.toMutableSet()
+                        val innerClassesToKeep = mutableSetOf<String>()
                         val writer = ClassWriter(0)
                         val remapper = ClassRemapper(writer, object : Remapper() {
                             override fun map(internalName: String): String =
@@ -150,13 +150,11 @@ class JvmAbiOutputExtension(
                             override fun visitEnd() {}
                         }, 0)
 
-                        innerClassesToKeep.retainAll(innerClassInfos.keys)
-                        for (name in innerClassesToKeep.toList()) {
-                            var info = innerClassInfos[name]
-                            while (info != null) {
-                                info = info.outerName?.takeIf(innerClassesToKeep::add)?.let(innerClassInfos::get)
-                            }
-                        }
+                        innerClassesToKeep.addInnerClasses(innerClassInfos, internalName)
+                        innerClassesToKeep.addOuterClasses(innerClassInfos)
+
+                        // Output classes in sorted order so that changes in original ordering due to method bodies, etc.
+                        // don't affect the ABI JAR.
                         for (name in innerClassesToKeep.sorted()) {
                             innerClassInfos[name]?.let { writer.visitInnerClass(it.name, it.outerName, it.innerName, it.access) }
                         }
@@ -170,5 +168,28 @@ class JvmAbiOutputExtension(
 
             return metadata + classFiles
         }
+
+        // Outer class infos for a class and all classes transitively nested in it (that are public ABI)
+        // should be kept in its own class file even if the classes are otherwise unused.
+        private fun MutableSet<String>.addInnerClasses(innerClassInfos: Map<String, InnerClassInfo>, internalName: String) {
+            val innerClassesByOuterName = innerClassInfos.values.groupBy { it.outerName }
+            val stack = mutableListOf(internalName)
+            while (stack.isNotEmpty()) {
+                val next = stack.removeLast()
+                add(next)
+                // Classes form a tree by nesting, so none of the children have been visited yet.
+                innerClassesByOuterName[next]?.mapNotNullTo(stack) { it.name.takeIf(abiClassInfos::contains) }
+            }
+        }
+
+        // For every class A.B, if its outer class info is kept then so should be A's.
+        private fun MutableSet<String>.addOuterClasses(innerClassInfos: Map<String, InnerClassInfo>) {
+            for (name in toList()) {
+                var info = innerClassInfos[name]
+                while (info != null) {
+                    info = info.outerName?.takeIf(::add)?.let(innerClassInfos::get)
+                }
+            }
+        }
     }
 }

plugins/jvm-abi-gen/testData/content/innerClasses/innerClasses.kt

@@ -2,13 +2,19 @@
 class InAnnotation {}
 
 class InPublicMethod {
-    fun foo(x: Outer.Middle.Inner) {}
+    fun foo(x: Outer.Middle.Inner): Class<*> = OuterKt.MiddleKt.InnerKt::class.java
 }
 
 class InPrivateMethod {
-    private fun foo(x: Outer.Middle.Inner) {}
+    private fun foo(x: Outer.Middle.Inner): Class<*> = OuterKt.MiddleKt.InnerKt::class.java
 }
 
 class InInlineMethod {
-    inline fun foo(): Class<*> = Outer.Middle.Inner::class.java
+    inline fun foo(x: Outer.Middle.Inner): Class<*> = OuterKt.MiddleKt.InnerKt::class.java
+}
+
+class OuterKt {
+    class MiddleKt {
+        class InnerKt
+    }
 }

plugins/jvm-abi-gen/testData/content/innerClasses/signatures.txt

@@ -15,8 +15,10 @@ public final class InInlineMethod {
     // source: 'innerClasses.kt'
     public inner class Outer$Middle
     public inner class Outer$Middle$Inner
+    public final inner class OuterKt$MiddleKt
+    public final inner class OuterKt$MiddleKt$InnerKt
     public method <init>(): void
-    public final @org.jetbrains.annotations.NotNull method foo(): java.lang.Class
+    public final @org.jetbrains.annotations.NotNull method foo(@org.jetbrains.annotations.NotNull p0: Outer$Middle$Inner): java.lang.Class
 }
 @kotlin.Metadata
 public final class InPrivateMethod {
@@ -29,7 +31,7 @@ public final class InPublicMethod {
     public inner class Outer$Middle
     public inner class Outer$Middle$Inner
     public method <init>(): void
-    public final method foo(@org.jetbrains.annotations.NotNull p0: Outer$Middle$Inner): void
+    public final @org.jetbrains.annotations.NotNull method foo(@org.jetbrains.annotations.NotNull p0: Outer$Middle$Inner): java.lang.Class
 }
 public class Outer$Middle$Inner {
     // source: 'Outer.java'
@@ -48,3 +50,23 @@ public class Outer {
     public inner class Outer$Middle
     public method <init>(): void
 }
+@kotlin.Metadata
+public final class OuterKt$MiddleKt$InnerKt {
+    // source: 'innerClasses.kt'
+    public final inner class OuterKt$MiddleKt
+    public final inner class OuterKt$MiddleKt$InnerKt
+    public method <init>(): void
+}
+@kotlin.Metadata
+public final class OuterKt$MiddleKt {
+    // source: 'innerClasses.kt'
+    public final inner class OuterKt$MiddleKt
+    public final inner class OuterKt$MiddleKt$InnerKt
+    public method <init>(): void
+}
+@kotlin.Metadata
+public final class OuterKt {
+    // source: 'innerClasses.kt'
+    public final inner class OuterKt$MiddleKt
+    public method <init>(): void
+}