FIR LT: Introduce source file abstraction, carry it from parsing to IR

along with source lines mapping, allows to "emulate" usage of the PSI files which allows to extract source file and line mapping info on every stage from source element. It makes sense to use this mapping for the error reporting too.
2022-02-19 21:07:21 +01:00
parent bd60d4b2a6
commit 03cbfea737
45 changed files with 539 additions and 239 deletions
@@ -5,7 +5,6 @@

 package org.jetbrains.kotlin.cli.common.fir

-import org.jetbrains.kotlin.SequentialFilePositionFinder
 import org.jetbrains.kotlin.cli.common.messages.*
 import org.jetbrains.kotlin.diagnostics.DiagnosticUtils
 import org.jetbrains.kotlin.diagnostics.KtDiagnostic
@@ -13,7 +12,9 @@ import org.jetbrains.kotlin.diagnostics.KtPsiDiagnostic
 import org.jetbrains.kotlin.diagnostics.Severity
 import org.jetbrains.kotlin.diagnostics.impl.BaseDiagnosticsCollector
 import org.jetbrains.kotlin.diagnostics.rendering.RootDiagnosticRendererFactory
+import java.io.Closeable
 import java.io.File
+import java.io.InputStreamReader

 object FirDiagnosticsCompilerResultsReporter {
    fun reportToMessageCollector(
@@ -55,6 +56,7 @@ object FirDiagnosticsCompilerResultsReporter {
                            )
                        }
                        else -> {
+                            // TODO: bring KtSourceFile and KtSourceFileLinesMapping here and rewrite reporting via it to avoid code duplication
                            // NOTE: SequentialPositionFinder relies on the ascending order of the input offsets, so the code relies
                            // on the the appropriate sorting above
                            // Also the end offset is ignored, as it is irrelevant for the CLI reporting
@@ -132,4 +134,91 @@ object FirDiagnosticsCompilerResultsReporter {

 fun BaseDiagnosticsCollector.reportToMessageCollector(messageCollector: MessageCollector, renderDiagnosticName: Boolean) {
    FirDiagnosticsCompilerResultsReporter.reportToMessageCollector(this, messageCollector, renderDiagnosticName)
-}
+}
+
+private class KtSourceFilePos(val line: Int, val column: Int, val lineContent: String?) {
+
+    // NOTE: This method is used for presenting positions to the user
+    override fun toString(): String = if (line < 0) "(offset: $column line unknown)" else "($line,$column)"
+
+    companion object {
+        val NONE = KtSourceFilePos(-1, -1, null)
+    }
+}
+
+private class SequentialFilePositionFinder(file: File) : Closeable {
+
+    private var reader: InputStreamReader = file.reader(/* TODO: select proper charset */)
+
+    private var currentLineContent: String? = null
+    private val buffer = CharArray(255)
+    private var bufLength = -1
+    private var bufPos = 0
+    private var endOfStream = false
+    private var skipNextLf = false
+
+    private var charsRead = 0
+    private var currentLine = 0
+
+    // assuming that if called multiple times, calls should be sorted by ascending offset
+    fun findNextPosition(offset: Int, withLineContents: Boolean = true): KtSourceFilePos {
+        assert(offset >= charsRead - (currentLineContent?.length ?: 0))
+
+        fun posInCurrentLine(): KtSourceFilePos? {
+            val col = offset - (charsRead - currentLineContent!!.length - 1)/* beginning of line offset */ + 1 /* col is 1-based */
+            return if (col <= currentLineContent!!.length)
+                KtSourceFilePos(currentLine, col, if (withLineContents) currentLineContent else null)
+            else null
+        }
+
+        if (offset < charsRead) {
+            return posInCurrentLine()!!
+        }
+
+        while (true) {
+            if (currentLineContent == null) {
+                currentLineContent = readNextLine()
+            }
+
+            posInCurrentLine()?.let { return@findNextPosition it }
+
+            if (endOfStream) return KtSourceFilePos(-1, offset, if (withLineContents) currentLineContent else null)
+
+            currentLineContent = null
+        }
+    }
+
+    private fun readNextLine() = buildString {
+        while (true) {
+            if (bufPos >= bufLength) {
+                bufLength = reader.read(buffer)
+                bufPos = 0
+                if (bufLength < 0) {
+                    endOfStream = true
+                    break
+                }
+            } else {
+                val c = buffer[bufPos++]
+                charsRead++
+                when {
+                    c == '\n' && skipNextLf -> {
+                        skipNextLf = false
+                    }
+                    c == '\n' || c == '\r' -> {
+                        currentLine++
+                        skipNextLf = c == '\r'
+                        break
+                    }
+                    else -> {
+                        append(c)
+                        skipNextLf = false
+                    }
+                }
+            }
+        }
+    }
+
+    override fun close() {
+        reader.close()
+    }
+}
@@ -7,12 +7,17 @@ package org.jetbrains.kotlin.cli.jvm.compiler

 import com.intellij.openapi.project.Project
 import com.intellij.openapi.vfs.StandardFileSystems
+import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.openapi.vfs.VirtualFileManager
 import com.intellij.openapi.vfs.VirtualFileSystem
 import com.intellij.psi.PsiElementFinder
 import com.intellij.psi.PsiFile
 import com.intellij.psi.search.GlobalSearchScope
 import com.intellij.psi.search.ProjectScope
+import org.jetbrains.kotlin.KtIoFileSourceFile
+import org.jetbrains.kotlin.KtPsiSourceFile
+import org.jetbrains.kotlin.KtSourceFile
+import org.jetbrains.kotlin.KtVirtualFileSourceFile
 import org.jetbrains.kotlin.asJava.finder.JavaElementFinder
 import org.jetbrains.kotlin.fir.FirModuleData
 import org.jetbrains.kotlin.fir.FirSession
@@ -65,16 +70,33 @@ open class VfsBasedProjectEnvironment(
        psiFinderExtensionPoint.registerExtension(FirJavaElementFinder(firSession, project), project)
    }

+    private fun List<VirtualFile>.toSearchScope(allowOutOfProjectRoots: Boolean) =
+        takeIf { it.isNotEmpty() }
+            ?.let {
+                if (allowOutOfProjectRoots) GlobalSearchScope.filesWithLibrariesScope(project, it)
+                else GlobalSearchScope.filesWithoutLibrariesScope(project, it)
+            }
+            ?: GlobalSearchScope.EMPTY_SCOPE
+
    override fun getSearchScopeByIoFiles(files: Iterable<File>, allowOutOfProjectRoots: Boolean): AbstractProjectFileSearchScope =
        PsiBasedProjectFileSearchScope(
            files
                .mapNotNull { localFileSystem.findFileByPath(it.absolutePath) }
-                .toList()
-                .takeIf { it.isNotEmpty() }
-                ?.let {
-                    if (allowOutOfProjectRoots) GlobalSearchScope.filesWithLibrariesScope(project, it)
-                    else GlobalSearchScope.filesWithoutLibrariesScope(project, it)
-                } ?: GlobalSearchScope.EMPTY_SCOPE
+                .toSearchScope(allowOutOfProjectRoots)
+        )
+
+    override fun getSearchScopeBySourceFiles(files: Iterable<KtSourceFile>, allowOutOfProjectRoots: Boolean): AbstractProjectFileSearchScope =
+        PsiBasedProjectFileSearchScope(
+            files
+                .mapNotNull {
+                    when (it) {
+                        is KtPsiSourceFile -> it.psiFile.virtualFile
+                        is KtVirtualFileSourceFile -> it.virtualFile
+                        is KtIoFileSourceFile -> localFileSystem.findFileByPath(it.file.absolutePath)
+                        else -> null // TODO: find out whether other use cases should be supported
+                    }
+                }
+                .toSearchScope(allowOutOfProjectRoots)
        )

    override fun getSearchScopeByDirectories(directories: Iterable<File>): AbstractProjectFileSearchScope =
@@ -108,8 +130,6 @@ open class VfsBasedProjectEnvironment(
        fileSearchScope: AbstractProjectFileSearchScope
    ) = FirJavaFacade(firSession, baseModuleData, project.createJavaClassFinder(fileSearchScope.asPsiSearchScope()))

-    override fun getFileText(filePath: String): String? =
-        localFileSystem.findFileByPath(filePath)?.inputStream?.reader(Charsets.UTF_8)?.readText()
 }

 private fun AbstractProjectFileSearchScope.asPsiSearchScope() =
@@ -17,6 +17,8 @@ import com.intellij.openapi.vfs.VirtualFileSystem
 import com.intellij.psi.PsiManager
 import com.intellij.psi.search.GlobalSearchScope
 import com.intellij.util.io.URLUtil
+import org.jetbrains.kotlin.KtSourceFile
+import org.jetbrains.kotlin.KtVirtualFileSourceFile
 import org.jetbrains.kotlin.analyzer.common.CommonPlatformAnalyzerServices
 import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension
 import org.jetbrains.kotlin.backend.jvm.JvmGeneratorExtensionsImpl
@@ -105,13 +107,12 @@ fun compileModulesUsingFrontendIrAndLightTree(
        val moduleConfiguration = compilerConfiguration.copy().applyModuleProperties(module, buildFile).apply {
            addAll(JVMConfigurationKeys.FRIEND_PATHS, module.getFriendPaths())
        }
-        val platformSources = linkedSetOf<File>()
-        val commonSources = linkedSetOf<File>()
+        val platformSources = linkedSetOf<KtSourceFile>()
+        val commonSources = linkedSetOf<KtSourceFile>()

        // !!
        compilerConfiguration.kotlinSourceRoots.forAllFiles(compilerConfiguration, projectEnvironment.project) { virtualFile, isCommon ->
-            val file = File(virtualFile.canonicalPath ?: virtualFile.path)
-            if (!file.isFile) error("TODO: better error: file not found $virtualFile")
+            val file = KtVirtualFileSourceFile(virtualFile)
            if (isCommon) commonSources.add(file)
            else platformSources.add(file)
        }
@@ -261,14 +262,14 @@ fun compileModuleToAnalyzedFir(
    diagnosticsReporter: DiagnosticReporter,
    performanceManager: CommonCompilerPerformanceManager?
 ): ModuleCompilerAnalyzedOutput {
-    var sourcesScope = environment.projectEnvironment.getSearchScopeByIoFiles(input.platformSources) //!!
+    var sourcesScope = environment.projectEnvironment.getSearchScopeBySourceFiles(input.platformSources)
    val sessionProvider = FirProjectSessionProvider()
    val extendedAnalysisMode = input.configuration.getBoolean(CommonConfigurationKeys.USE_FIR_EXTENDED_CHECKERS)

    val commonSession = runIf(
        input.commonSources.isNotEmpty() && input.configuration.languageVersionSettings.supportsFeature(LanguageFeature.MultiPlatformProjects)
    ) {
-        val commonSourcesScope = environment.projectEnvironment.getSearchScopeByIoFiles(input.commonSources) //!!
+        val commonSourcesScope = environment.projectEnvironment.getSearchScopeBySourceFiles(input.commonSources)
        sourcesScope -= commonSourcesScope
        createSession(
            "${input.targetId.name}-common",
@@ -310,12 +311,11 @@ fun compileModuleToAnalyzedFir(
    // raw fir
    val commonRawFir = commonSession?.buildFirViaLightTree(
        input.commonSources,
-        environment.projectEnvironment,
        diagnosticsReporter,
        countFilesAndLines
    )
    val rawFir =
-        session.buildFirViaLightTree(input.platformSources, environment.projectEnvironment, diagnosticsReporter, countFilesAndLines)
+        session.buildFirViaLightTree(input.platformSources, diagnosticsReporter, countFilesAndLines)

    // resolution
    commonSession?.apply {
@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.cli.jvm.compiler.pipeline

+import org.jetbrains.kotlin.KtSourceFile
 import org.jetbrains.kotlin.backend.jvm.JvmGeneratorExtensionsImpl
 import org.jetbrains.kotlin.codegen.state.GenerationState
 import org.jetbrains.kotlin.config.CompilerConfiguration
@@ -27,9 +28,9 @@ import java.io.File
 data class ModuleCompilerInput(
    val targetId: TargetId,
    val commonPlatform: TargetPlatform,
-    val commonSources: Collection<File>,
+    val commonSources: Collection<KtSourceFile>,
    val platform: TargetPlatform,
-    val platformSources: Collection<File>,
+    val platformSources: Collection<KtSourceFile>,
    val configuration: CompilerConfiguration,
    val friendFirModules: Collection<FirModuleData> = emptyList()
 )