FIR: introduce diagnostic NESTED_CLASS_NOT_ALLOWED

compiler/fir/analysis-tests/testData/resolve/diagnostics/localAnnotationClass.kt

@@ -3,6 +3,6 @@ fun foo() {
 
     @Ann class Local {
         <!LOCAL_ANNOTATION_CLASS_ERROR{LT}!>// There should also be NESTED_CLASS_NOT_ALLOWED report here.
-        <!LOCAL_ANNOTATION_CLASS_ERROR{PSI}!>annotation class Nested<!><!>
+        <!LOCAL_ANNOTATION_CLASS_ERROR{PSI}!>annotation <!NESTED_CLASS_NOT_ALLOWED!>class Nested<!><!><!>
     }
 }

compiler/fir/analysis-tests/testData/resolve/diagnostics/localEntitytNotAllowed.kt

@@ -8,7 +8,7 @@ object A {
     val a = object : Any() {
         <!LOCAL_OBJECT_NOT_ALLOWED!>object D<!> {
             <!LOCAL_OBJECT_NOT_ALLOWED!>object G<!>
-            <!LOCAL_INTERFACE_NOT_ALLOWED!>interface Z<!>
+            <!LOCAL_INTERFACE_NOT_ALLOWED, NESTED_CLASS_NOT_ALLOWED!>interface Z<!>
         }
 
         <!LOCAL_INTERFACE_NOT_ALLOWED!>interface Y<!>
@@ -17,7 +17,7 @@ object A {
     fun b() {
         <!LOCAL_OBJECT_NOT_ALLOWED!>object E<!> {
             <!LOCAL_OBJECT_NOT_ALLOWED!>object F<!>
-            <!LOCAL_INTERFACE_NOT_ALLOWED!>interface M<!>
+            <!LOCAL_INTERFACE_NOT_ALLOWED, NESTED_CLASS_NOT_ALLOWED!>interface M<!>
         }
 
         <!LOCAL_INTERFACE_NOT_ALLOWED!>interface N<!>

compiler/fir/checkers/checkers-component-generator/src/org/jetbrains/kotlin/fir/checkers/generator/diagnostics/FirDiagnosticsList.kt

@@ -42,6 +42,9 @@ object DIAGNOSTICS_LIST : DiagnosticList() {
         val VARIABLE_EXPECTED by error<FirSourceElement, PsiElement>()
         val RETURN_NOT_ALLOWED by error<FirSourceElement, PsiElement>()
         val DELEGATION_IN_INTERFACE by error<FirSourceElement, PsiElement>()
+        val NESTED_CLASS_NOT_ALLOWED by error<FirSourceElement, KtNamedDeclaration>(PositioningStrategy.DECLARATION_NAME) {
+            parameter<String>("declaration")
+        }
     }
 
     val UNRESOLVED by object : DiagnosticGroup("Unresolved") {

compiler/fir/checkers/gen/org/jetbrains/kotlin/fir/analysis/diagnostics/FirErrors.kt

@@ -61,6 +61,7 @@ object FirErrors {
     val VARIABLE_EXPECTED by error0<FirSourceElement, PsiElement>()
     val RETURN_NOT_ALLOWED by error0<FirSourceElement, PsiElement>()
     val DELEGATION_IN_INTERFACE by error0<FirSourceElement, PsiElement>()
+    val NESTED_CLASS_NOT_ALLOWED by error1<FirSourceElement, KtNamedDeclaration, String>(SourceElementPositioningStrategies.DECLARATION_NAME)
 
     // Unresolved
     val HIDDEN by error1<FirSourceElement, PsiElement, AbstractFirBasedSymbol<*>>(SourceElementPositioningStrategies.REFERENCE_BY_QUALIFIED)

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/declaration/FirNestedClassChecker.kt

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.fir.analysis.checkers.declaration
+
+import org.jetbrains.kotlin.descriptors.ClassKind
+import org.jetbrains.kotlin.fir.analysis.checkers.context.CheckerContext
+import org.jetbrains.kotlin.fir.analysis.diagnostics.DiagnosticReporter
+import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.NESTED_CLASS_NOT_ALLOWED
+import org.jetbrains.kotlin.fir.analysis.diagnostics.reportOn
+import org.jetbrains.kotlin.fir.declarations.*
+
+// No need to visit anonymous object since an anonymous object is always inner. This aligns with
+// compiler/frontend/src/org/jetbrains/kotlin/resolve/ModifiersChecker.java:198
+object FirNestedClassChecker : FirRegularClassChecker() {
+    override fun check(declaration: FirRegularClass, context: CheckerContext, reporter: DiagnosticReporter) {
+        // Local enums / objects / companion objects are handled with different diagnostic codes.
+        if ((declaration.classKind.isSingleton || declaration.classKind == ClassKind.ENUM_CLASS) && declaration.isLocal) return
+        val containingDeclaration = context.containingDeclarations.lastOrNull() ?: return
+
+        when (containingDeclaration) {
+            is FirRegularClass -> {
+                if (!declaration.isInner && (containingDeclaration.isInner || containingDeclaration.isLocal)) {
+                    reporter.reportOn(declaration.source, NESTED_CLASS_NOT_ALLOWED, declaration.description, context)
+                }
+            }
+            is FirClass<*> -> {
+                // Since 1.3, enum entries can contain inner classes only.
+                // Companion objects are reported with code WRONG_MODIFIER_CONTAINING_DECLARATION instead
+                if (containingDeclaration.classKind == ClassKind.ENUM_ENTRY && !declaration.isInner && !declaration.isCompanion) {
+                    reporter.reportOn(declaration.source, NESTED_CLASS_NOT_ALLOWED, declaration.description, context)
+                }
+            }
+        }
+    }
+
+    // Note: here we don't differentiate anonymous object like in FE1.0
+    // (org.jetbrains.kotlin.resolve.ModifiersChecker.DetailedClassKind) because this case has been ruled out in the first place.
+    private val FirRegularClass.description: String
+        get() = when (classKind) {
+            ClassKind.CLASS -> "Class"
+            ClassKind.INTERFACE -> "Interface"
+            ClassKind.ENUM_CLASS -> "Enum class"
+            ClassKind.ENUM_ENTRY -> "Enum entry"
+            ClassKind.ANNOTATION_CLASS -> "Annotation class"
+            ClassKind.OBJECT -> if (this.isCompanion) "Companion object" else "Object"
+        }
+}

compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/diagnostics/FirDefaultErrorMessages.kt

@@ -98,6 +98,7 @@ import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.LOCAL_INTERFACE_N
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.LOCAL_OBJECT_NOT_ALLOWED
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.MANY_COMPANION_OBJECTS
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.MISSING_VAL_ON_ANNOTATION_PARAMETER
+import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.NESTED_CLASS_NOT_ALLOWED
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.MULTIPLE_VARARG_PARAMETERS
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.NONE_APPLICABLE
 import org.jetbrains.kotlin.fir.analysis.diagnostics.FirErrors.NON_ABSTRACT_FUNCTION_WITH_NO_BODY
@@ -211,6 +212,7 @@ class FirDefaultErrorMessages : DefaultErrorMessages.Extension {
             map.put(VARIABLE_EXPECTED, "Variable expected")
             map.put(RETURN_NOT_ALLOWED, "'return' is not allowed here")
             map.put(DELEGATION_IN_INTERFACE, "Interfaces cannot use delegation")
+            map.put(NESTED_CLASS_NOT_ALLOWED, "{0} is not allowed here", TO_STRING)
 
             // Unresolved
             map.put(HIDDEN, "Symbol {0} is invisible", SYMBOL)

compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/checkers/CommonDeclarationCheckers.kt

@@ -60,6 +60,7 @@ object CommonDeclarationCheckers : DeclarationCheckers() {
         FirTypeParametersInObjectChecker,
         FirMemberFunctionChecker,
         FirMemberPropertyChecker,
+        FirNestedClassChecker,
     )
 
     override val constructorCheckers: Set<FirConstructorChecker> = setOf(

compiler/fir/raw-fir/psi2fir/src/org/jetbrains/kotlin/fir/builder/RawFirBuilder.kt

@@ -793,7 +793,11 @@ class RawFirBuilder(
         override fun visitClassOrObject(classOrObject: KtClassOrObject, data: Unit): FirElement {
             return withChildClassName(
                 classOrObject.nameAsSafeName,
-                classOrObject.isLocal || classOrObject.getStrictParentOfType<KtEnumEntry>() != null
+                classOrObject.isLocal
+                        // TODO: currently enum entry initializer is represented in FIR as an FirAnonymousObject. Because of this, all
+                        //  nested declarations are now marked local. This causes the FirNestedClassChecker to ignore some invalid programs.
+                        // See KT-45115
+                        || classOrObject.getStrictParentOfType<KtEnumEntry>() != null
             ) {
                 val classKind = when (classOrObject) {
                     is KtObjectDeclaration -> ClassKind.OBJECT