[FIR, LT] Report syntax error by traversing LT after parsing

#KT-57756 Fixed

compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/LightTree2Fir.kt

@@ -7,14 +7,14 @@ package org.jetbrains.kotlin.fir.lightTree
 
 import com.intellij.lang.LighterASTNode
 import com.intellij.lang.PsiBuilderFactory
+import com.intellij.lang.impl.PsiBuilderImpl
+import com.intellij.openapi.util.Ref
+import com.intellij.psi.TokenType
 import com.intellij.util.diff.FlyweightCapableTreeStructure
 import org.jetbrains.kotlin.KtIoFileSourceFile
 import org.jetbrains.kotlin.KtSourceFile
 import org.jetbrains.kotlin.KtSourceFileLinesMapping
-import org.jetbrains.kotlin.config.LanguageVersionSettings
-import org.jetbrains.kotlin.diagnostics.DiagnosticContext
 import org.jetbrains.kotlin.diagnostics.DiagnosticReporter
-import org.jetbrains.kotlin.diagnostics.KtDiagnostic
 import org.jetbrains.kotlin.fir.FirSession
 import org.jetbrains.kotlin.fir.declarations.FirFile
 import org.jetbrains.kotlin.fir.languageVersionSettings
@@ -30,16 +30,44 @@ import java.nio.file.Path
 class LightTree2Fir(
     val session: FirSession,
     private val scopeProvider: FirScopeProvider,
-    private val diagnosticsReporter: DiagnosticReporter? = null
+    private val diagnosticsReporter: DiagnosticReporter? = null,
 ) {
     companion object {
         private val parserDefinition = KotlinParserDefinition()
         private fun makeLexer() = KotlinLexer()
 
-        fun buildLightTree(code: CharSequence): FlyweightCapableTreeStructure<LighterASTNode> {
+        fun buildLightTree(
+            code: CharSequence,
+            errorListener: LightTreeParsingErrorListener?,
+        ): FlyweightCapableTreeStructure<LighterASTNode> {
             val builder = PsiBuilderFactory.getInstance().createBuilder(parserDefinition, makeLexer(), code)
-            return KotlinLightParser.parse(builder)
+            return KotlinLightParser.parse(builder).also {
+                if (errorListener != null) reportErrors(it.root, it, errorListener)
+            }
         }
+
+        private fun reportErrors(
+            node: LighterASTNode,
+            tree: FlyweightCapableTreeStructure<LighterASTNode>,
+            errorListener: LightTreeParsingErrorListener,
+            ref: Ref<Array<LighterASTNode?>> = Ref<Array<LighterASTNode?>>(),
+        ) {
+            tree.getChildren(node, ref)
+            val kidsArray = ref.get() ?: return
+
+            for (kid in kidsArray) {
+                if (kid == null) break
+                val tokenType = kid.tokenType
+                if (tokenType == TokenType.ERROR_ELEMENT) {
+                    val message = PsiBuilderImpl.getErrorMessage(kid)
+                    errorListener.onError(kid.startOffset, kid.endOffset, message)
+                }
+
+                ref.set(null)
+                reportErrors(kid, tree, errorListener, ref)
+            }
+        }
+
     }
 
     fun buildFirFile(path: Path): FirFile {
@@ -57,21 +85,20 @@ class LightTree2Fir(
     fun buildFirFile(
         lightTree: FlyweightCapableTreeStructure<LighterASTNode>,
         sourceFile: KtSourceFile,
-        linesMapping: KtSourceFileLinesMapping
-    ): FirFile =
-        DeclarationsConverter(
-            session, scopeProvider, lightTree, diagnosticsReporter = diagnosticsReporter,
-            diagnosticContext = makeDiagnosticContext(sourceFile.path)
-        ).convertFile(lightTree.root, sourceFile, linesMapping)
+        linesMapping: KtSourceFileLinesMapping,
+    ): FirFile {
+        return DeclarationsConverter(session, scopeProvider, lightTree)
+            .convertFile(lightTree.root, sourceFile, linesMapping)
+    }
 
-    fun buildFirFile(code: CharSequence, sourceFile: KtSourceFile, linesMapping: KtSourceFileLinesMapping): FirFile =
-        buildFirFile(buildLightTree(code), sourceFile, linesMapping)
+    fun buildFirFile(code: CharSequence, sourceFile: KtSourceFile, linesMapping: KtSourceFileLinesMapping): FirFile {
+        val errorListener = makeErrorListener(sourceFile)
+        val lightTree = buildLightTree(code, errorListener)
+        return buildFirFile(lightTree, sourceFile, linesMapping)
+    }
 
-    private fun makeDiagnosticContext(path: String?) =
-        if (diagnosticsReporter == null) null else object : DiagnosticContext {
-            override val containingFilePath = path
-            override val languageVersionSettings: LanguageVersionSettings get() = session.languageVersionSettings
-            override fun isDiagnosticSuppressed(diagnostic: KtDiagnostic): Boolean = false
-        }
+    private fun makeErrorListener(sourceFile: KtSourceFile): LightTreeParsingErrorListener? {
+        val diagnosticsReporter = diagnosticsReporter ?: return null
+        return diagnosticsReporter.toKotlinParsingErrorListener(sourceFile, session.languageVersionSettings)
+    }
 }
-

compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/LightTreeParsingErrorListener.kt

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.fir.lightTree
+
+import org.jetbrains.kotlin.KtOffsetsOnlySourceElement
+import org.jetbrains.kotlin.KtSourceFile
+import org.jetbrains.kotlin.config.LanguageVersionSettings
+import org.jetbrains.kotlin.diagnostics.DiagnosticContext
+import org.jetbrains.kotlin.diagnostics.DiagnosticReporter
+import org.jetbrains.kotlin.diagnostics.KtDiagnostic
+import org.jetbrains.kotlin.diagnostics.reportOn
+import org.jetbrains.kotlin.fir.builder.FirSyntaxErrors
+
+fun interface LightTreeParsingErrorListener {
+    fun onError(startOffset: Int, endOffset: Int, message: String?)
+}
+
+fun DiagnosticReporter.toKotlinParsingErrorListener(
+    sourceFile: KtSourceFile,
+    languageVersionSettings: LanguageVersionSettings
+): LightTreeParsingErrorListener {
+    val diagnosticContext = object : DiagnosticContext {
+        override val containingFilePath = sourceFile.path
+        override val languageVersionSettings: LanguageVersionSettings get() = languageVersionSettings
+        override fun isDiagnosticSuppressed(diagnostic: KtDiagnostic): Boolean = false
+    }
+    return LightTreeParsingErrorListener { startOffset, endOffset, message ->
+        reportOn(
+            KtOffsetsOnlySourceElement(startOffset, endOffset),
+            FirSyntaxErrors.SYNTAX,
+            message.orEmpty(),
+            diagnosticContext,
+        )
+    }
+}

compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/converter/BaseConverter.kt

@@ -29,8 +29,6 @@ abstract class BaseConverter(
 ) : BaseFirBuilder<LighterASTNode>(baseSession, context) {
     protected val implicitType = FirImplicitTypeRefImplWithoutSource
 
-    protected open fun reportSyntaxError(node: LighterASTNode) {}
-
     override fun LighterASTNode.toFirSourceElement(kind: KtFakeSourceElementKind?): KtLightSourceElement {
         val startOffset = tree.getStartOffset(this)
         val endOffset = tree.getEndOffset(this)
@@ -169,17 +167,12 @@ abstract class BaseConverter(
         return getChildrenAsArray().firstOrNull()
     }
 
-    @OptIn(ExperimentalContracts::class)
     protected inline fun LighterASTNode.forEachChildren(vararg skipTokens: KtToken, f: (LighterASTNode) -> Unit) {
         val kidsArray = this.getChildrenAsArray()
         for (kid in kidsArray) {
             if (kid == null) break
             val tokenType = kid.tokenType
-            if (COMMENTS.contains(tokenType) || tokenType == WHITE_SPACE || tokenType == SEMICOLON || tokenType in skipTokens) continue
-            if (tokenType == TokenType.ERROR_ELEMENT) {
-                reportSyntaxError(kid)
-                continue
-            }
+            if (COMMENTS.contains(tokenType) || tokenType == WHITE_SPACE || tokenType == SEMICOLON || tokenType in skipTokens || tokenType == TokenType.ERROR_ELEMENT) continue
             f(kid)
         }
     }
@@ -191,11 +184,7 @@ abstract class BaseConverter(
         for (kid in kidsArray) {
             if (kid == null) break
             val tokenType = kid.tokenType
-            if (COMMENTS.contains(tokenType) || tokenType == WHITE_SPACE || tokenType == SEMICOLON) continue
-            if (tokenType == TokenType.ERROR_ELEMENT) {
-                reportSyntaxError(kid)
-                continue
-            }
+            if (COMMENTS.contains(tokenType) || tokenType == WHITE_SPACE || tokenType == SEMICOLON || tokenType == TokenType.ERROR_ELEMENT) continue
             f(kid, container)
         }
 

compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/converter/DeclarationsConverter.kt

@@ -6,7 +6,6 @@
 package org.jetbrains.kotlin.fir.lightTree.converter
 
 import com.intellij.lang.LighterASTNode
-import com.intellij.lang.impl.PsiBuilderImpl
 import com.intellij.psi.TokenType
 import com.intellij.util.diff.FlyweightCapableTreeStructure
 import org.jetbrains.kotlin.*
@@ -19,9 +18,6 @@ import org.jetbrains.kotlin.descriptors.Visibilities
 import org.jetbrains.kotlin.descriptors.Visibility
 import org.jetbrains.kotlin.descriptors.annotations.AnnotationUseSiteTarget
 import org.jetbrains.kotlin.descriptors.annotations.AnnotationUseSiteTarget.*
-import org.jetbrains.kotlin.diagnostics.DiagnosticContext
-import org.jetbrains.kotlin.diagnostics.DiagnosticReporter
-import org.jetbrains.kotlin.diagnostics.reportOn
 import org.jetbrains.kotlin.fir.*
 import org.jetbrains.kotlin.fir.builder.*
 import org.jetbrains.kotlin.fir.contracts.FirContractDescription
@@ -40,7 +36,6 @@ import org.jetbrains.kotlin.fir.diagnostics.DiagnosticKind
 import org.jetbrains.kotlin.fir.expressions.*
 import org.jetbrains.kotlin.fir.expressions.builder.*
 import org.jetbrains.kotlin.fir.expressions.impl.FirSingleExpressionBlock
-import org.jetbrains.kotlin.fir.lightTree.LightTree2Fir
 import org.jetbrains.kotlin.fir.lightTree.fir.*
 import org.jetbrains.kotlin.fir.lightTree.fir.modifier.Modifier
 import org.jetbrains.kotlin.fir.lightTree.fir.modifier.TypeModifier
@@ -70,22 +65,10 @@ class DeclarationsConverter(
     internal val baseScopeProvider: FirScopeProvider,
     tree: FlyweightCapableTreeStructure<LighterASTNode>,
     context: Context<LighterASTNode> = Context(),
-    private val diagnosticsReporter: DiagnosticReporter? = null,
-    private val diagnosticContext: DiagnosticContext? = null,
 ) : BaseConverter(session, tree, context) {
 
     private val expressionConverter = ExpressionsConverter(session, tree, this, context)
 
-    override fun reportSyntaxError(node: LighterASTNode) {
-        val message = PsiBuilderImpl.getErrorMessage(node)
-        diagnosticsReporter?.reportOn(
-            node.toFirSourceElement(),
-            FirSyntaxErrors.SYNTAX,
-            message ?: "Unspecified",
-            diagnosticContext!!,
-        )
-    }
-
     /**
      * [org.jetbrains.kotlin.parsing.KotlinParsing.parseFile]
      * [org.jetbrains.kotlin.parsing.KotlinParsing.parsePreamble]

compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/TotalKotlinTest.kt

@@ -42,7 +42,7 @@ class TotalKotlinTest : AbstractRawFirBuilderTestCase() {
         text: CharSequence, sourceFile: KtSourceFile, linesMapping: KtSourceFileLinesMapping
     ) {
         if (onlyLightTree) {
-            val lightTree = LightTree2Fir.buildLightTree(text)
+            val lightTree = LightTree2Fir.buildLightTree(text, null)
             DebugUtil.lightTreeToString(lightTree, false)
         } else {
             val firFile = converter.buildFirFile(text, sourceFile, linesMapping)

compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/SourceFileProvider.kt

@@ -9,6 +9,8 @@ import com.intellij.openapi.project.Project
 import com.intellij.openapi.vfs.StandardFileSystems
 import com.intellij.psi.PsiManager
 import org.jetbrains.kotlin.KtInMemoryTextSourceFile
+import org.jetbrains.kotlin.KtSourceFile
+import org.jetbrains.kotlin.fir.lightTree.LightTreeParsingErrorListener
 import org.jetbrains.kotlin.fir.lightTree.LightTree2Fir
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.sourceFiles.LightTreeFile
@@ -110,20 +112,26 @@ fun SourceFileProvider.getKtFilesForSourceFiles(testFiles: Collection<TestFile>,
     }.toMap()
 }
 
-fun SourceFileProvider.getLightTreeKtFileForSourceFile(testFile: TestFile): LightTreeFile {
+fun SourceFileProvider.getLightTreeKtFileForSourceFile(
+    testFile: TestFile,
+    errorListener: (KtSourceFile) -> LightTreeParsingErrorListener?
+): LightTreeFile {
     val shortName = testFile.toLightTreeShortName()
     val sourceFile = KtInMemoryTextSourceFile(shortName, "/$shortName", getContentOfSourceFile(testFile))
     val linesMapping = sourceFile.text.toSourceLinesMapping()
-    val lightTree = LightTree2Fir.buildLightTree(sourceFile.text)
+    val lightTree = LightTree2Fir.buildLightTree(sourceFile.text, errorListener(sourceFile))
     return LightTreeFile(lightTree, sourceFile, linesMapping)
 }
 
 fun TestFile.toLightTreeShortName() = name.substringAfterLast('/').substringAfterLast('\\')
 
-fun SourceFileProvider.getLightTreeFilesForSourceFiles(testFiles: Collection<TestFile>): Map<TestFile, LightTreeFile> {
+fun SourceFileProvider.getLightTreeFilesForSourceFiles(
+    testFiles: Collection<TestFile>,
+    errorListener: (KtSourceFile) -> LightTreeParsingErrorListener?
+): Map<TestFile, LightTreeFile> {
     return testFiles.mapNotNull {
         if (!it.isKtFile) return@mapNotNull null
-        it to getLightTreeKtFileForSourceFile(it)
+        it to getLightTreeKtFileForSourceFile(it, errorListener)
     }.toMap()
 }
 

compiler/testData/diagnostics/tests/annotations/kt1860-negative.fir.kt

@@ -4,7 +4,7 @@ fun foo(@<!UNRESOLVED_REFERENCE!>varargs<!> f : Int) {}
 var bar : Int = 1
   set(@<!UNRESOLVED_REFERENCE!>varargs<!> v) {}
 
-val x : (Int) -> Int = <!INITIALIZER_TYPE_MISMATCH{LT}!>{@<!UNRESOLVED_REFERENCE!>varargs<!> x <!SYNTAX!>: Int -> x<!>}<!>
+val x : (Int) -> Int = {@<!UNRESOLVED_REFERENCE!>varargs<!> x <!SYNTAX!>: Int -> x<!>}
 
 class Hello(@<!UNRESOLVED_REFERENCE!>varargs<!> args: Any) {
 }

compiler/testData/diagnostics/tests/annotations/kt1860-positive.fir.kt

@@ -8,7 +8,7 @@ fun foo(@test f : Int) {}
 var bar : Int = 1
     set(@test v) {}
 
-val x : (Int) -> Int = <!INITIALIZER_TYPE_MISMATCH{LT}!>{@test x <!SYNTAX!>: Int -> x<!>}<!> // todo fix parser annotation on lambda parameter
+val x : (Int) -> Int = {@test x <!SYNTAX!>: Int -> x<!>} // todo fix parser annotation on lambda parameter
 
 class Hello(@test args: Any) {
 }

compiler/testData/diagnostics/tests/incompleteCode/diagnosticWithSyntaxError/checkBackingFieldException.fir.kt

@@ -1,19 +0,0 @@
-// COMPARE_WITH_LIGHT_TREE
-package h
-
-class Square() {
-  var size : Double =
-  <!UNRESOLVED_REFERENCE!>set<!>(<!UNRESOLVED_REFERENCE!>value<!>) {
-    //in LT this LAMBDA_EXPRESSION get parsed lazyly, but doesn't got anywhere in FIR tree (as property doesn't have place for it)
-    <!SYNTAX{PSI}!>$area<!> <!SYNTAX{PSI}!>= size * size<!>
-  }
-
-  <!MUST_BE_INITIALIZED_OR_BE_ABSTRACT!>var area : Double<!>
-  private set
-}
-
-fun main() {
-  val s = Square()
-
-  s.size = 2.0
-}

compiler/testData/diagnostics/tests/incompleteCode/diagnosticWithSyntaxError/checkBackingFieldException.kt

@@ -1,10 +1,10 @@
+// FIR_IDENTICAL
 // COMPARE_WITH_LIGHT_TREE
 package h
 
 class Square() {
   var size : Double =
   <!UNRESOLVED_REFERENCE!>set<!>(<!UNRESOLVED_REFERENCE!>value<!>) {
-    //in LT this LAMBDA_EXPRESSION get parsed lazyly, but doesn't got anywhere in FIR tree (as property doesn't have place for it)
     <!SYNTAX!>$area<!> <!SYNTAX!>= size * size<!>
   }
 

compiler/testData/diagnostics/tests/incompleteCode/incompleteAssignment.fir.kt

@@ -8,7 +8,7 @@ fun sum(a : IntArray) : Int {
     <!UNRESOLVED_REFERENCE!>res<!> = 0
     for (e in a)
         <!UNRESOLVED_REFERENCE!>res<!> +=<!SYNTAX!><!>
-<!NO_RETURN_IN_FUNCTION_WITH_BLOCK_BODY{LT}!>}<!>
+}
 fun main() {
     test(0)
     test(1, 1)

compiler/testData/diagnostics/tests/regressions/kt328.fir.kt

@@ -25,5 +25,5 @@ val z = <!TYPECHECKER_HAS_RUN_INTO_RECURSIVE_PROBLEM!>z<!>
 fun block(f : () -> Unit) = f()
 
 fun bar3() = block{ <!UNRESOLVED_REFERENCE!>foo3<!>() // <-- missing closing curly bracket
-fun foo3() = block{ <!TYPECHECKER_HAS_RUN_INTO_RECURSIVE_PROBLEM!>bar3()<!> }<!SYNTAX{PSI}!><!>
+fun foo3() = block{ <!TYPECHECKER_HAS_RUN_INTO_RECURSIVE_PROBLEM!>bar3()<!> }<!SYNTAX!><!>
 

compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFrontendFacade.kt

@@ -29,6 +29,7 @@ import org.jetbrains.kotlin.fir.checkers.registerExtendedCommonCheckers
 import org.jetbrains.kotlin.fir.deserialization.ModuleDataProvider
 import org.jetbrains.kotlin.fir.extensions.FirExtensionRegistrar
 import org.jetbrains.kotlin.fir.java.FirProjectSessionProvider
+import org.jetbrains.kotlin.fir.lightTree.toKotlinParsingErrorListener
 import org.jetbrains.kotlin.fir.session.FirCommonSessionFactory
 import org.jetbrains.kotlin.fir.session.FirJvmSessionFactory
 import org.jetbrains.kotlin.fir.session.FirNativeSessionFactory
@@ -283,7 +284,14 @@ open class FirFrontendFacade(
         val parser = module.directives.singleValue(FirDiagnosticsDirectives.FIR_PARSER)
 
         val (ktFiles, lightTreeFiles) = when (parser) {
-            FirParser.LightTree -> emptyList<KtFile>() to testServices.sourceFileProvider.getLightTreeFilesForSourceFiles(module.files).values
+            FirParser.LightTree -> {
+                emptyList<KtFile>() to testServices.sourceFileProvider.getLightTreeFilesForSourceFiles(module.files) {
+                    testServices.lightTreeSyntaxDiagnosticsReporterHolder?.reporter?.toKotlinParsingErrorListener(
+                        it,
+                        module.languageVersionSettings
+                    )
+                }.values
+            }
             FirParser.Psi -> testServices.sourceFileProvider.getKtFilesForSourceFiles(module.files, project).values to emptyList()
         }
 

compiler/tests/org/jetbrains/kotlin/lightTree/LightTreeParsingTest.kt

@@ -33,7 +33,7 @@ class LightTreeParsingTest {
             val (code, mapping) = ByteArrayInputStream(toByteArray()).reader().readSourceFileWithMapping()
             val positionFinder = SequentialPositionFinder(ByteArrayInputStream(toByteArray()).reader())
             val linePositions =
-                LightTree2Fir.buildLightTree(code).getChildrenAsArray()
+                LightTree2Fir.buildLightTree(code, null).getChildrenAsArray()
                     .mapNotNull { it?.startOffset }
                     .map {
                         val nextPos = positionFinder.findNextPosition(it)