Fix several bugs in serialization of inner generic classes

- Interner was working incorrectly with parents
- nested classes were serialized in codegen out of any context

 #KT-5660 Fixed

compiler/backend/src/org/jetbrains/jet/codegen/ImplementationBodyCodegen.java

@@ -232,7 +232,8 @@ public class ImplementationBodyCodegen extends ClassBodyCodegen {
 
         if (state.getClassBuilderMode() != ClassBuilderMode.FULL) return;
 
-        DescriptorSerializer serializer = new DescriptorSerializer(new JavaSerializerExtension(v.getSerializationBindings()));
+        DescriptorSerializer serializer =
+                DescriptorSerializer.create(descriptor, new JavaSerializerExtension(v.getSerializationBindings()));
 
         ProtoBuf.Class classProto = serializer.classProto(descriptor).build();
 

compiler/backend/src/org/jetbrains/jet/codegen/PackageCodegen.java

@@ -272,7 +272,7 @@ public class PackageCodegen {
             if (file.isScript()) return;
         }
 
-        DescriptorSerializer serializer = new DescriptorSerializer(new JavaSerializerExtension(bindings));
+        DescriptorSerializer serializer = DescriptorSerializer.createTopLevel(new JavaSerializerExtension(bindings));
         Collection<PackageFragmentDescriptor> packageFragments = Lists.newArrayList();
         ContainerUtil.addIfNotNull(packageFragments, packageFragment);
         ContainerUtil.addIfNotNull(packageFragments, compiledPackageFragment);

compiler/builtins-serializer/src/BuiltInsSerializer.kt

@@ -110,7 +110,7 @@ public class BuiltInsSerializer(val out: PrintStream?) {
         // TODO: perform some kind of validation? At the moment not possible because DescriptorValidator is in compiler-tests
         // DescriptorValidator.validate(packageView)
 
-        val serializer = DescriptorSerializer(SerializerExtension.DEFAULT)
+        val serializer = DescriptorSerializer.createTopLevel(SerializerExtension.DEFAULT)
 
         val classNames = ArrayList<Name>()
         val classifierDescriptors = DescriptorSerializer.sort(packageView.getMemberScope().getDescriptors(DescriptorKindFilter.CLASSIFIERS))

compiler/testData/loadJava/compiledKotlin/nested/deepInnerGeneric.kt

@@ -0,0 +1,13 @@
+package test
+
+class A<TA> {
+    inner class B<TB> {
+        inner class C<TC> {
+            inner class D<TD> {
+                fun <P1, P2, P3, P4> foo(p1: P1, p2: P2, p3: P3, p4: P4): Nothing = null!!
+
+                fun bar(ta: TA, tb: TB, tc: TC, td: TD): A<TA>.B<TB>.C<TC>.D<TD> = foo<TA, TB, TC, TD>(ta, tb, tc, td)
+            }
+        }
+    }
+}

compiler/testData/loadJava/compiledKotlin/nested/deepInnerGeneric.txt

@@ -0,0 +1,19 @@
+package test
+
+internal final class A</*0*/ TA> {
+    /*primary*/ public constructor A</*0*/ TA>()
+
+    internal final inner class B</*0*/ TB> {
+        /*primary*/ public constructor B</*0*/ TB>()
+
+        internal final inner class C</*0*/ TC> {
+            /*primary*/ public constructor C</*0*/ TC>()
+
+            internal final inner class D</*0*/ TD> {
+                /*primary*/ public constructor D</*0*/ TD>()
+                internal final fun bar(/*0*/ ta: TA, /*1*/ tb: TB, /*2*/ tc: TC, /*3*/ td: TD): test.A.B.C.D<TD>
+                internal final fun </*0*/ P1, /*1*/ P2, /*2*/ P3, /*3*/ P4> foo(/*0*/ p1: P1, /*1*/ p2: P2, /*2*/ p3: P3, /*3*/ p4: P4): kotlin.Nothing
+            }
+        }
+    }
+}

compiler/testData/loadJava/compiledKotlin/nested/innerClassReferencesOuterTP.kt

@@ -0,0 +1,5 @@
+package test
+
+class InnerClassReferencesOuterTP<P> {
+    inner class Inner<Q : P>
+}

compiler/testData/loadJava/compiledKotlin/nested/innerClassReferencesOuterTP.txt

@@ -0,0 +1,9 @@
+package test
+
+internal final class InnerClassReferencesOuterTP</*0*/ P> {
+    /*primary*/ public constructor InnerClassReferencesOuterTP</*0*/ P>()
+
+    internal final inner class Inner</*0*/ Q : P> {
+        /*primary*/ public constructor Inner</*0*/ Q : P>()
+    }
+}

compiler/testData/loadJava/compiledKotlin/nested/membersReferenceOuterTP.kt

@@ -0,0 +1,11 @@
+package test
+
+class MembersReferenceOuterTP<P> {
+    inner class Inner {
+        fun f<Q : P>() {}
+        fun g(p: P): P = null!!
+
+        val v: P = null!!
+        val <Q : P> w: Q = null!!
+    }
+}

compiler/testData/loadJava/compiledKotlin/nested/membersReferenceOuterTP.txt

@@ -0,0 +1,15 @@
+package test
+
+internal final class MembersReferenceOuterTP</*0*/ P> {
+    /*primary*/ public constructor MembersReferenceOuterTP</*0*/ P>()
+
+    internal final inner class Inner {
+        /*primary*/ public constructor Inner()
+        internal final val v: P
+            internal final fun <get-v>(): P
+        internal final val </*0*/ Q : P> w: Q
+            internal final fun <get-w>(): Q
+        internal final fun </*0*/ Q : P> f(): kotlin.Unit
+        internal final fun g(/*0*/ p: P): P
+    }
+}

compiler/tests/org/jetbrains/jet/jvm/compiler/LoadJavaTestGenerated.java

@@ -1801,7 +1801,7 @@ public class LoadJavaTestGenerated extends AbstractLoadJavaTest {
 
     @TestMetadata("compiler/testData/loadJava/compiledKotlin")
     @TestDataPath("$PROJECT_ROOT")
-    @InnerTestClasses({CompiledKotlin.Annotations.class, CompiledKotlin.Class.class, CompiledKotlin.ClassFun.class, CompiledKotlin.ClassObject.class, CompiledKotlin.Constructor.class, CompiledKotlin.DataClass.class, CompiledKotlin.Enum.class, CompiledKotlin.FromLoadJava.class, CompiledKotlin.Fun.class, CompiledKotlin.Inline.class, CompiledKotlin.MemberOrder.class, CompiledKotlin.PlatformTypes.class, CompiledKotlin.Prop.class, CompiledKotlin.Type.class, CompiledKotlin.Visibility.class})
+    @InnerTestClasses({CompiledKotlin.Annotations.class, CompiledKotlin.Class.class, CompiledKotlin.ClassFun.class, CompiledKotlin.ClassObject.class, CompiledKotlin.Constructor.class, CompiledKotlin.DataClass.class, CompiledKotlin.Enum.class, CompiledKotlin.FromLoadJava.class, CompiledKotlin.Fun.class, CompiledKotlin.Inline.class, CompiledKotlin.MemberOrder.class, CompiledKotlin.Nested.class, CompiledKotlin.PlatformTypes.class, CompiledKotlin.Prop.class, CompiledKotlin.Type.class, CompiledKotlin.Visibility.class})
     @RunWith(JUnit3RunnerWithInners.class)
     public static class CompiledKotlin extends AbstractLoadJavaTest {
         public void testAllFilesPresentInCompiledKotlin() throws Exception {
@@ -4176,6 +4176,33 @@ public class LoadJavaTestGenerated extends AbstractLoadJavaTest {
             }
         }
 
+        @TestMetadata("compiler/testData/loadJava/compiledKotlin/nested")
+        @TestDataPath("$PROJECT_ROOT")
+        @RunWith(JUnit3RunnerWithInners.class)
+        public static class Nested extends AbstractLoadJavaTest {
+            public void testAllFilesPresentInNested() throws Exception {
+                JetTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("compiler/testData/loadJava/compiledKotlin/nested"), Pattern.compile("^(.+)\\.kt$"), true);
+            }
+
+            @TestMetadata("deepInnerGeneric.kt")
+            public void testDeepInnerGeneric() throws Exception {
+                String fileName = JetTestUtils.navigationMetadata("compiler/testData/loadJava/compiledKotlin/nested/deepInnerGeneric.kt");
+                doTestCompiledKotlin(fileName);
+            }
+
+            @TestMetadata("innerClassReferencesOuterTP.kt")
+            public void testInnerClassReferencesOuterTP() throws Exception {
+                String fileName = JetTestUtils.navigationMetadata("compiler/testData/loadJava/compiledKotlin/nested/innerClassReferencesOuterTP.kt");
+                doTestCompiledKotlin(fileName);
+            }
+
+            @TestMetadata("membersReferenceOuterTP.kt")
+            public void testMembersReferenceOuterTP() throws Exception {
+                String fileName = JetTestUtils.navigationMetadata("compiler/testData/loadJava/compiledKotlin/nested/membersReferenceOuterTP.kt");
+                doTestCompiledKotlin(fileName);
+            }
+        }
+
         @TestMetadata("compiler/testData/loadJava/compiledKotlin/platformTypes")
         @TestDataPath("$PROJECT_ROOT")
         @RunWith(JUnit3RunnerWithInners.class)

compiler/tests/org/jetbrains/jet/lang/resolve/lazy/LazyResolveRecursiveComparingTestGenerated.java

@@ -33,7 +33,7 @@ import java.util.regex.Pattern;
 public class LazyResolveRecursiveComparingTestGenerated extends AbstractLazyResolveRecursiveComparingTest {
     @TestMetadata("compiler/testData/loadJava/compiledKotlin")
     @TestDataPath("$PROJECT_ROOT")
-    @InnerTestClasses({CompiledKotlin.Annotations.class, CompiledKotlin.Class.class, CompiledKotlin.ClassFun.class, CompiledKotlin.ClassObject.class, CompiledKotlin.Constructor.class, CompiledKotlin.DataClass.class, CompiledKotlin.Enum.class, CompiledKotlin.FromLoadJava.class, CompiledKotlin.Fun.class, CompiledKotlin.Inline.class, CompiledKotlin.MemberOrder.class, CompiledKotlin.PlatformTypes.class, CompiledKotlin.Prop.class, CompiledKotlin.Type.class, CompiledKotlin.Visibility.class})
+    @InnerTestClasses({CompiledKotlin.Annotations.class, CompiledKotlin.Class.class, CompiledKotlin.ClassFun.class, CompiledKotlin.ClassObject.class, CompiledKotlin.Constructor.class, CompiledKotlin.DataClass.class, CompiledKotlin.Enum.class, CompiledKotlin.FromLoadJava.class, CompiledKotlin.Fun.class, CompiledKotlin.Inline.class, CompiledKotlin.MemberOrder.class, CompiledKotlin.Nested.class, CompiledKotlin.PlatformTypes.class, CompiledKotlin.Prop.class, CompiledKotlin.Type.class, CompiledKotlin.Visibility.class})
     @RunWith(JUnit3RunnerWithInners.class)
     public static class CompiledKotlin extends AbstractLazyResolveRecursiveComparingTest {
         public void testAllFilesPresentInCompiledKotlin() throws Exception {
@@ -2408,6 +2408,33 @@ public class LazyResolveRecursiveComparingTestGenerated extends AbstractLazyReso
             }
         }
 
+        @TestMetadata("compiler/testData/loadJava/compiledKotlin/nested")
+        @TestDataPath("$PROJECT_ROOT")
+        @RunWith(JUnit3RunnerWithInners.class)
+        public static class Nested extends AbstractLazyResolveRecursiveComparingTest {
+            public void testAllFilesPresentInNested() throws Exception {
+                JetTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("compiler/testData/loadJava/compiledKotlin/nested"), Pattern.compile("^(.+)\\.kt$"), true);
+            }
+
+            @TestMetadata("deepInnerGeneric.kt")
+            public void testDeepInnerGeneric() throws Exception {
+                String fileName = JetTestUtils.navigationMetadata("compiler/testData/loadJava/compiledKotlin/nested/deepInnerGeneric.kt");
+                doTest(fileName);
+            }
+
+            @TestMetadata("innerClassReferencesOuterTP.kt")
+            public void testInnerClassReferencesOuterTP() throws Exception {
+                String fileName = JetTestUtils.navigationMetadata("compiler/testData/loadJava/compiledKotlin/nested/innerClassReferencesOuterTP.kt");
+                doTest(fileName);
+            }
+
+            @TestMetadata("membersReferenceOuterTP.kt")
+            public void testMembersReferenceOuterTP() throws Exception {
+                String fileName = JetTestUtils.navigationMetadata("compiler/testData/loadJava/compiledKotlin/nested/membersReferenceOuterTP.kt");
+                doTest(fileName);
+            }
+        }
+
         @TestMetadata("compiler/testData/loadJava/compiledKotlin/platformTypes")
         @TestDataPath("$PROJECT_ROOT")
         @RunWith(JUnit3RunnerWithInners.class)

core/serialization/src/org/jetbrains/jet/descriptors/serialization/DescriptorSerializer.java

@@ -37,16 +37,35 @@ public class DescriptorSerializer {
     private final Interner<TypeParameterDescriptor> typeParameters;
     private final SerializerExtension extension;
 
-    public DescriptorSerializer(@NotNull SerializerExtension extension) {
-        this(new NameTable(), new Interner<TypeParameterDescriptor>(), extension);
-    }
-
     private DescriptorSerializer(NameTable nameTable, Interner<TypeParameterDescriptor> typeParameters, SerializerExtension extension) {
         this.nameTable = nameTable;
         this.typeParameters = typeParameters;
         this.extension = extension;
     }
 
+    @NotNull
+    public static DescriptorSerializer createTopLevel(@NotNull SerializerExtension extension) {
+        return new DescriptorSerializer(new NameTable(), new Interner<TypeParameterDescriptor>(), extension);
+    }
+
+    @NotNull
+    public static DescriptorSerializer create(@NotNull ClassDescriptor descriptor, @NotNull SerializerExtension extension) {
+        DeclarationDescriptor container = descriptor.getContainingDeclaration();
+        DescriptorSerializer parentSerializer =
+                container instanceof ClassDescriptor
+                ? create((ClassDescriptor) container, extension)
+                : createTopLevel(extension);
+
+        // Calculate type parameter ids for the outer class beforehand, as it would've had happened if we were always
+        // serializing outer classes before nested classes.
+        // Otherwise our interner can get wrong ids because we may serialize classes in any order.
+        DescriptorSerializer serializer = parentSerializer.createChildSerializer();
+        for (TypeParameterDescriptor typeParameter : descriptor.getTypeConstructor().getParameters()) {
+            serializer.typeParameters.intern(typeParameter);
+        }
+        return serializer;
+    }
+
     private DescriptorSerializer createChildSerializer() {
         return new DescriptorSerializer(nameTable, new Interner<TypeParameterDescriptor>(typeParameters), extension);
     }
@@ -68,16 +87,14 @@ public class DescriptorSerializer {
 
         builder.setFqName(getClassId(classDescriptor));
 
-        DescriptorSerializer local = createChildSerializer();
-
         for (TypeParameterDescriptor typeParameterDescriptor : classDescriptor.getTypeConstructor().getParameters()) {
-            builder.addTypeParameter(local.typeParameter(typeParameterDescriptor));
+            builder.addTypeParameter(typeParameter(typeParameterDescriptor));
         }
 
         if (!KotlinBuiltIns.getInstance().isSpecialClassWithNoSupertypes(classDescriptor)) {
             // Special classes (Any, Nothing) have no supertypes
             for (JetType supertype : classDescriptor.getTypeConstructor().getSupertypes()) {
-                builder.addSupertype(local.type(supertype));
+                builder.addSupertype(type(supertype));
             }
         }
 
@@ -88,7 +105,7 @@ public class DescriptorSerializer {
             }
             else {
                 ProtoBuf.Class.PrimaryConstructor.Builder constructorBuilder = ProtoBuf.Class.PrimaryConstructor.newBuilder();
-                constructorBuilder.setData(local.callableProto(primaryConstructor));
+                constructorBuilder.setData(callableProto(primaryConstructor));
                 builder.setPrimaryConstructor(constructorBuilder);
             }
         }
@@ -99,7 +116,7 @@ public class DescriptorSerializer {
             if (descriptor instanceof CallableMemberDescriptor) {
                 CallableMemberDescriptor member = (CallableMemberDescriptor) descriptor;
                 if (member.getKind() == CallableMemberDescriptor.Kind.FAKE_OVERRIDE) continue;
-                builder.addMember(local.callableProto(member));
+                builder.addMember(callableProto(member));
             }
         }
 

core/serialization/src/org/jetbrains/jet/descriptors/serialization/Interner.java

@@ -32,7 +32,7 @@ public final class Interner<T> {
 
     public Interner(Interner<T> parent, @NotNull TObjectHashingStrategy<T> hashing) {
         this.parent = parent;
-        this.firstIndex = parent == null ? 0 : parent.all.size();
+        this.firstIndex = parent != null ? parent.all.size() + parent.firstIndex : 0;
         this.interned = new TObjectIntHashMap<T>(hashing);
     }
 
@@ -40,30 +40,39 @@ public final class Interner<T> {
         this(null, hashing);
     }
 
+    @SuppressWarnings("unchecked")
     public Interner(@Nullable Interner<T> parent) {
-        //noinspection unchecked
         this(parent, TObjectHashingStrategy.CANONICAL);
     }
 
     public Interner() {
-        //noinspection unchecked
-        this((Interner) null);
+        this((Interner<T>) null);
     }
 
-    public int intern(@NotNull T obj) {
-        assert parent == null || parent.all.size() == firstIndex : "Parent changed in parallel with child: indexes will be wrong";
-        if (parent != null && parent.interned.contains(obj)) {
-            return parent.intern(obj);
+    private int find(@NotNull T obj) {
+        assert parent == null || parent.all.size() + parent.firstIndex == firstIndex :
+                "Parent changed in parallel with child: indexes will be wrong";
+        if (parent != null) {
+            int index = parent.find(obj);
+            if (index >= 0) return index;
         }
         if (interned.contains(obj)) {
             return interned.get(obj);
         }
-        int index = firstIndex + interned.size();
+        return -1;
+    }
+
+    public int intern(@NotNull T obj) {
+        int index = find(obj);
+        if (index >= 0) return index;
+
+        index = firstIndex + interned.size();
         interned.put(obj, index);
         all.add(obj);
         return index;
     }
 
+    @NotNull
     public List<T> getAllInternedObjects() {
         return all;
     }

idea/tests/org/jetbrains/jet/plugin/stubs/LazyResolveByStubTestGenerated.java

@@ -30,7 +30,7 @@ import java.util.regex.Pattern;
 @SuppressWarnings("all")
 @TestMetadata("compiler/testData/loadJava/compiledKotlin")
 @TestDataPath("$PROJECT_ROOT")
-@InnerTestClasses({LazyResolveByStubTestGenerated.Annotations.class, LazyResolveByStubTestGenerated.Class.class, LazyResolveByStubTestGenerated.ClassFun.class, LazyResolveByStubTestGenerated.ClassObject.class, LazyResolveByStubTestGenerated.Constructor.class, LazyResolveByStubTestGenerated.DataClass.class, LazyResolveByStubTestGenerated.Enum.class, LazyResolveByStubTestGenerated.FromLoadJava.class, LazyResolveByStubTestGenerated.Fun.class, LazyResolveByStubTestGenerated.Inline.class, LazyResolveByStubTestGenerated.MemberOrder.class, LazyResolveByStubTestGenerated.PlatformTypes.class, LazyResolveByStubTestGenerated.Prop.class, LazyResolveByStubTestGenerated.Type.class, LazyResolveByStubTestGenerated.Visibility.class})
+@InnerTestClasses({LazyResolveByStubTestGenerated.Annotations.class, LazyResolveByStubTestGenerated.Class.class, LazyResolveByStubTestGenerated.ClassFun.class, LazyResolveByStubTestGenerated.ClassObject.class, LazyResolveByStubTestGenerated.Constructor.class, LazyResolveByStubTestGenerated.DataClass.class, LazyResolveByStubTestGenerated.Enum.class, LazyResolveByStubTestGenerated.FromLoadJava.class, LazyResolveByStubTestGenerated.Fun.class, LazyResolveByStubTestGenerated.Inline.class, LazyResolveByStubTestGenerated.MemberOrder.class, LazyResolveByStubTestGenerated.Nested.class, LazyResolveByStubTestGenerated.PlatformTypes.class, LazyResolveByStubTestGenerated.Prop.class, LazyResolveByStubTestGenerated.Type.class, LazyResolveByStubTestGenerated.Visibility.class})
 @RunWith(JUnit3RunnerWithInners.class)
 public class LazyResolveByStubTestGenerated extends AbstractLazyResolveByStubTest {
     public void testAllFilesPresentInCompiledKotlin() throws Exception {
@@ -2405,6 +2405,33 @@ public class LazyResolveByStubTestGenerated extends AbstractLazyResolveByStubTes
         }
     }
 
+    @TestMetadata("compiler/testData/loadJava/compiledKotlin/nested")
+    @TestDataPath("$PROJECT_ROOT")
+    @RunWith(JUnit3RunnerWithInners.class)
+    public static class Nested extends AbstractLazyResolveByStubTest {
+        public void testAllFilesPresentInNested() throws Exception {
+            JetTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("compiler/testData/loadJava/compiledKotlin/nested"), Pattern.compile("^(.+)\\.kt$"), true);
+        }
+
+        @TestMetadata("deepInnerGeneric.kt")
+        public void testDeepInnerGeneric() throws Exception {
+            String fileName = JetTestUtils.navigationMetadata("compiler/testData/loadJava/compiledKotlin/nested/deepInnerGeneric.kt");
+            doTest(fileName);
+        }
+
+        @TestMetadata("innerClassReferencesOuterTP.kt")
+        public void testInnerClassReferencesOuterTP() throws Exception {
+            String fileName = JetTestUtils.navigationMetadata("compiler/testData/loadJava/compiledKotlin/nested/innerClassReferencesOuterTP.kt");
+            doTest(fileName);
+        }
+
+        @TestMetadata("membersReferenceOuterTP.kt")
+        public void testMembersReferenceOuterTP() throws Exception {
+            String fileName = JetTestUtils.navigationMetadata("compiler/testData/loadJava/compiledKotlin/nested/membersReferenceOuterTP.kt");
+            doTest(fileName);
+        }
+    }
+
     @TestMetadata("compiler/testData/loadJava/compiledKotlin/platformTypes")
     @TestDataPath("$PROJECT_ROOT")
     @RunWith(JUnit3RunnerWithInners.class)