From 03cbfea73783300af23c8d1040cfccd732dc8d2f Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Sat, 19 Feb 2022 21:07:21 +0100 Subject: [PATCH] FIR LT: Introduce source file abstraction, carry it from parsing to IR along with source lines mapping, allows to "emulate" usage of the PSI files which allows to extract source file and line mapping info on every stage from source element. It makes sense to use this mapping for the error reporting too. --- .../based/LowLevelFirAnalyzerFacade.kt | 2 +- .../FirDiagnosticsCompilerResultsReporter.kt | 93 ++++++++++++- .../compiler/VfsBasedProjectEnvironment.kt | 36 +++-- .../jvm/compiler/pipeline/compilerPipeline.kt | 16 +-- .../compiler/pipeline/compilerPipelineData.kt | 5 +- .../checkers/context/CheckerContext.kt | 2 +- .../jetbrains/kotlin/fir/pipeline/buildFir.kt | 22 ++- .../environment/AbstractProjectEnvironment.kt | 5 +- .../kotlin/fir/backend/Fir2IrConverter.kt | 15 ++- .../kotlin/fir/lightTree/LightTree2Fir.kt | 42 +++--- .../converter/DeclarationsConverter.kt | 7 +- .../kotlin/fir/lightTree/TotalKotlinTest.kt | 18 ++- .../fir/lightTree/compare/TreesCompareTest.kt | 19 ++- .../kotlin/fir/builder/RawFirBuilder.kt | 3 +- .../kotlin/fir/declarations/FirFile.kt | 5 +- .../declarations/builder/FirFileBuilder.kt | 8 +- .../fir/declarations/impl/FirFileImpl.kt | 5 +- .../fir/tree/generator/NodeConfigurator.kt | 3 +- .../kotlin/fir/tree/generator/Types.kt | 4 + .../org/jetbrains/kotlin/KtSourceElement.kt | 90 ------------- .../src/org/jetbrains/kotlin/KtSourceFile.kt | 57 ++++++++ .../kotlin/KtSourceFileLinesMapping.kt | 126 ++++++++++++++++++ .../IncrementalFirJvmCompilerRunner.kt | 25 ++-- .../jetbrains/kotlin/backend/common/Utils.kt | 11 -- .../kotlin/backend/jvm/ir/JvmIrUtils.kt | 2 +- .../kotlin/ir/util/AdditionalIrUtils.kt | 21 +-- .../kotlin/sourceFiles/LightTreeFile.kt | 17 +++ .../test/services/SourceFileProvider.kt | 11 +- .../classic/ClassicJvmBackendFacade.kt | 3 +- .../backend/handlers/IrTextDumpHandler.kt | 7 +- .../test/backend/handlers/JvmBoxRunner.kt | 2 +- .../test/backend/handlers/SMAPDumpHandler.kt | 5 +- .../kotlin/test/backend/ir/IrBackendInput.kt | 4 +- .../test/backend/ir/JvmIrBackendFacade.kt | 16 ++- .../classic/ClassicFrontend2IrConverter.kt | 3 +- .../frontend/fir/Fir2IrResultsConverter.kt | 4 +- .../test/frontend/fir/FirFrontendFacade.kt | 2 +- .../test/frontend/fir/FirOutputArtifact.kt | 4 +- .../kotlin/test/model/ResultingArtifacts.kt | 7 +- ...nFunctionForBlackBoxTestsSourceProvider.kt | 13 +- .../jetbrains/kotlin/codegen/SMAPTestUtil.kt | 3 +- .../fir/AbstractFirBaseDiagnosticsTest.kt | 9 +- .../jetbrains/kotlin/fir/FirResolveBench.kt | 12 +- .../kotlin/codegen/GenerationUtils.kt | 2 +- .../kotlin/fir}/FirAnalyzerFacade.kt | 12 +- 45 files changed, 539 insertions(+), 239 deletions(-) create mode 100644 compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceFile.kt create mode 100644 compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceFileLinesMapping.kt create mode 100644 compiler/test-infrastructure-utils/tests/org/jetbrains/kotlin/sourceFiles/LightTreeFile.kt rename compiler/{fir/entrypoint/src/org/jetbrains/kotlin/fir/analysis => tests-compiler-utils/tests/org/jetbrains/kotlin/fir}/FirAnalyzerFacade.kt (94%) diff --git a/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/compiler/based/LowLevelFirAnalyzerFacade.kt b/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/compiler/based/LowLevelFirAnalyzerFacade.kt index 7590f1195a0..e8edb52e864 100644 --- a/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/compiler/based/LowLevelFirAnalyzerFacade.kt +++ b/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/compiler/based/LowLevelFirAnalyzerFacade.kt @@ -8,8 +8,8 @@ package org.jetbrains.kotlin.analysis.low.level.api.fir.compiler.based import org.jetbrains.kotlin.analysis.low.level.api.fir.api.DiagnosticCheckerFilter import org.jetbrains.kotlin.analysis.low.level.api.fir.api.LLFirModuleResolveState import org.jetbrains.kotlin.analysis.low.level.api.fir.api.collectDiagnosticsForFile -import org.jetbrains.kotlin.fir.analysis.AbstractFirAnalyzerFacade import org.jetbrains.kotlin.diagnostics.KtDiagnostic +import org.jetbrains.kotlin.fir.AbstractFirAnalyzerFacade import org.jetbrains.kotlin.fir.backend.Fir2IrResult import org.jetbrains.kotlin.fir.declarations.FirFile import org.jetbrains.kotlin.fir.declarations.FirResolvePhase diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/common/fir/FirDiagnosticsCompilerResultsReporter.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/common/fir/FirDiagnosticsCompilerResultsReporter.kt index 9827a107deb..f8e09bc01d6 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/common/fir/FirDiagnosticsCompilerResultsReporter.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/common/fir/FirDiagnosticsCompilerResultsReporter.kt @@ -5,7 +5,6 @@ package org.jetbrains.kotlin.cli.common.fir -import org.jetbrains.kotlin.SequentialFilePositionFinder import org.jetbrains.kotlin.cli.common.messages.* import org.jetbrains.kotlin.diagnostics.DiagnosticUtils import org.jetbrains.kotlin.diagnostics.KtDiagnostic @@ -13,7 +12,9 @@ import org.jetbrains.kotlin.diagnostics.KtPsiDiagnostic import org.jetbrains.kotlin.diagnostics.Severity import org.jetbrains.kotlin.diagnostics.impl.BaseDiagnosticsCollector import org.jetbrains.kotlin.diagnostics.rendering.RootDiagnosticRendererFactory +import java.io.Closeable import java.io.File +import java.io.InputStreamReader object FirDiagnosticsCompilerResultsReporter { fun reportToMessageCollector( @@ -55,6 +56,7 @@ object FirDiagnosticsCompilerResultsReporter { ) } else -> { + // TODO: bring KtSourceFile and KtSourceFileLinesMapping here and rewrite reporting via it to avoid code duplication // NOTE: SequentialPositionFinder relies on the ascending order of the input offsets, so the code relies // on the the appropriate sorting above // Also the end offset is ignored, as it is irrelevant for the CLI reporting @@ -132,4 +134,91 @@ object FirDiagnosticsCompilerResultsReporter { fun BaseDiagnosticsCollector.reportToMessageCollector(messageCollector: MessageCollector, renderDiagnosticName: Boolean) { FirDiagnosticsCompilerResultsReporter.reportToMessageCollector(this, messageCollector, renderDiagnosticName) -} \ No newline at end of file +} + +private class KtSourceFilePos(val line: Int, val column: Int, val lineContent: String?) { + + // NOTE: This method is used for presenting positions to the user + override fun toString(): String = if (line < 0) "(offset: $column line unknown)" else "($line,$column)" + + companion object { + val NONE = KtSourceFilePos(-1, -1, null) + } +} + +private class SequentialFilePositionFinder(file: File) : Closeable { + + private var reader: InputStreamReader = file.reader(/* TODO: select proper charset */) + + private var currentLineContent: String? = null + private val buffer = CharArray(255) + private var bufLength = -1 + private var bufPos = 0 + private var endOfStream = false + private var skipNextLf = false + + private var charsRead = 0 + private var currentLine = 0 + + // assuming that if called multiple times, calls should be sorted by ascending offset + fun findNextPosition(offset: Int, withLineContents: Boolean = true): KtSourceFilePos { + assert(offset >= charsRead - (currentLineContent?.length ?: 0)) + + fun posInCurrentLine(): KtSourceFilePos? { + val col = offset - (charsRead - currentLineContent!!.length - 1)/* beginning of line offset */ + 1 /* col is 1-based */ + return if (col <= currentLineContent!!.length) + KtSourceFilePos(currentLine, col, if (withLineContents) currentLineContent else null) + else null + } + + if (offset < charsRead) { + return posInCurrentLine()!! + } + + while (true) { + if (currentLineContent == null) { + currentLineContent = readNextLine() + } + + posInCurrentLine()?.let { return@findNextPosition it } + + if (endOfStream) return KtSourceFilePos(-1, offset, if (withLineContents) currentLineContent else null) + + currentLineContent = null + } + } + + private fun readNextLine() = buildString { + while (true) { + if (bufPos >= bufLength) { + bufLength = reader.read(buffer) + bufPos = 0 + if (bufLength < 0) { + endOfStream = true + break + } + } else { + val c = buffer[bufPos++] + charsRead++ + when { + c == '\n' && skipNextLf -> { + skipNextLf = false + } + c == '\n' || c == '\r' -> { + currentLine++ + skipNextLf = c == '\r' + break + } + else -> { + append(c) + skipNextLf = false + } + } + } + } + } + + override fun close() { + reader.close() + } +} diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/VfsBasedProjectEnvironment.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/VfsBasedProjectEnvironment.kt index d5532f37e67..0afedde160e 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/VfsBasedProjectEnvironment.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/VfsBasedProjectEnvironment.kt @@ -7,12 +7,17 @@ package org.jetbrains.kotlin.cli.jvm.compiler import com.intellij.openapi.project.Project import com.intellij.openapi.vfs.StandardFileSystems +import com.intellij.openapi.vfs.VirtualFile import com.intellij.openapi.vfs.VirtualFileManager import com.intellij.openapi.vfs.VirtualFileSystem import com.intellij.psi.PsiElementFinder import com.intellij.psi.PsiFile import com.intellij.psi.search.GlobalSearchScope import com.intellij.psi.search.ProjectScope +import org.jetbrains.kotlin.KtIoFileSourceFile +import org.jetbrains.kotlin.KtPsiSourceFile +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtVirtualFileSourceFile import org.jetbrains.kotlin.asJava.finder.JavaElementFinder import org.jetbrains.kotlin.fir.FirModuleData import org.jetbrains.kotlin.fir.FirSession @@ -65,16 +70,33 @@ open class VfsBasedProjectEnvironment( psiFinderExtensionPoint.registerExtension(FirJavaElementFinder(firSession, project), project) } + private fun List.toSearchScope(allowOutOfProjectRoots: Boolean) = + takeIf { it.isNotEmpty() } + ?.let { + if (allowOutOfProjectRoots) GlobalSearchScope.filesWithLibrariesScope(project, it) + else GlobalSearchScope.filesWithoutLibrariesScope(project, it) + } + ?: GlobalSearchScope.EMPTY_SCOPE + override fun getSearchScopeByIoFiles(files: Iterable, allowOutOfProjectRoots: Boolean): AbstractProjectFileSearchScope = PsiBasedProjectFileSearchScope( files .mapNotNull { localFileSystem.findFileByPath(it.absolutePath) } - .toList() - .takeIf { it.isNotEmpty() } - ?.let { - if (allowOutOfProjectRoots) GlobalSearchScope.filesWithLibrariesScope(project, it) - else GlobalSearchScope.filesWithoutLibrariesScope(project, it) - } ?: GlobalSearchScope.EMPTY_SCOPE + .toSearchScope(allowOutOfProjectRoots) + ) + + override fun getSearchScopeBySourceFiles(files: Iterable, allowOutOfProjectRoots: Boolean): AbstractProjectFileSearchScope = + PsiBasedProjectFileSearchScope( + files + .mapNotNull { + when (it) { + is KtPsiSourceFile -> it.psiFile.virtualFile + is KtVirtualFileSourceFile -> it.virtualFile + is KtIoFileSourceFile -> localFileSystem.findFileByPath(it.file.absolutePath) + else -> null // TODO: find out whether other use cases should be supported + } + } + .toSearchScope(allowOutOfProjectRoots) ) override fun getSearchScopeByDirectories(directories: Iterable): AbstractProjectFileSearchScope = @@ -108,8 +130,6 @@ open class VfsBasedProjectEnvironment( fileSearchScope: AbstractProjectFileSearchScope ) = FirJavaFacade(firSession, baseModuleData, project.createJavaClassFinder(fileSearchScope.asPsiSearchScope())) - override fun getFileText(filePath: String): String? = - localFileSystem.findFileByPath(filePath)?.inputStream?.reader(Charsets.UTF_8)?.readText() } private fun AbstractProjectFileSearchScope.asPsiSearchScope() = diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipeline.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipeline.kt index 0db0c85eef3..b3ca6807974 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipeline.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipeline.kt @@ -17,6 +17,8 @@ import com.intellij.openapi.vfs.VirtualFileSystem import com.intellij.psi.PsiManager import com.intellij.psi.search.GlobalSearchScope import com.intellij.util.io.URLUtil +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtVirtualFileSourceFile import org.jetbrains.kotlin.analyzer.common.CommonPlatformAnalyzerServices import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension import org.jetbrains.kotlin.backend.jvm.JvmGeneratorExtensionsImpl @@ -105,13 +107,12 @@ fun compileModulesUsingFrontendIrAndLightTree( val moduleConfiguration = compilerConfiguration.copy().applyModuleProperties(module, buildFile).apply { addAll(JVMConfigurationKeys.FRIEND_PATHS, module.getFriendPaths()) } - val platformSources = linkedSetOf() - val commonSources = linkedSetOf() + val platformSources = linkedSetOf() + val commonSources = linkedSetOf() // !! compilerConfiguration.kotlinSourceRoots.forAllFiles(compilerConfiguration, projectEnvironment.project) { virtualFile, isCommon -> - val file = File(virtualFile.canonicalPath ?: virtualFile.path) - if (!file.isFile) error("TODO: better error: file not found $virtualFile") + val file = KtVirtualFileSourceFile(virtualFile) if (isCommon) commonSources.add(file) else platformSources.add(file) } @@ -261,14 +262,14 @@ fun compileModuleToAnalyzedFir( diagnosticsReporter: DiagnosticReporter, performanceManager: CommonCompilerPerformanceManager? ): ModuleCompilerAnalyzedOutput { - var sourcesScope = environment.projectEnvironment.getSearchScopeByIoFiles(input.platformSources) //!! + var sourcesScope = environment.projectEnvironment.getSearchScopeBySourceFiles(input.platformSources) val sessionProvider = FirProjectSessionProvider() val extendedAnalysisMode = input.configuration.getBoolean(CommonConfigurationKeys.USE_FIR_EXTENDED_CHECKERS) val commonSession = runIf( input.commonSources.isNotEmpty() && input.configuration.languageVersionSettings.supportsFeature(LanguageFeature.MultiPlatformProjects) ) { - val commonSourcesScope = environment.projectEnvironment.getSearchScopeByIoFiles(input.commonSources) //!! + val commonSourcesScope = environment.projectEnvironment.getSearchScopeBySourceFiles(input.commonSources) sourcesScope -= commonSourcesScope createSession( "${input.targetId.name}-common", @@ -310,12 +311,11 @@ fun compileModuleToAnalyzedFir( // raw fir val commonRawFir = commonSession?.buildFirViaLightTree( input.commonSources, - environment.projectEnvironment, diagnosticsReporter, countFilesAndLines ) val rawFir = - session.buildFirViaLightTree(input.platformSources, environment.projectEnvironment, diagnosticsReporter, countFilesAndLines) + session.buildFirViaLightTree(input.platformSources, diagnosticsReporter, countFilesAndLines) // resolution commonSession?.apply { diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipelineData.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipelineData.kt index 493556159ad..9262543c686 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipelineData.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipelineData.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.cli.jvm.compiler.pipeline +import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.backend.jvm.JvmGeneratorExtensionsImpl import org.jetbrains.kotlin.codegen.state.GenerationState import org.jetbrains.kotlin.config.CompilerConfiguration @@ -27,9 +28,9 @@ import java.io.File data class ModuleCompilerInput( val targetId: TargetId, val commonPlatform: TargetPlatform, - val commonSources: Collection, + val commonSources: Collection, val platform: TargetPlatform, - val platformSources: Collection, + val platformSources: Collection, val configuration: CompilerConfiguration, val friendFirModules: Collection = emptyList() ) diff --git a/compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/context/CheckerContext.kt b/compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/context/CheckerContext.kt index 264d15a4860..6ebad6f8224 100644 --- a/compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/context/CheckerContext.kt +++ b/compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/context/CheckerContext.kt @@ -87,7 +87,7 @@ abstract class CheckerContext : MutableDiagnosticContext() { get() = session.languageVersionSettings override val containingFilePath: String? - get() = containingDeclarations.firstOrNull()?.let { (it as? FirFile)?.path } + get() = containingDeclarations.firstOrNull()?.let { (it as? FirFile)?.sourceFile?.path } } /** diff --git a/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/pipeline/buildFir.kt b/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/pipeline/buildFir.kt index 650c18cb078..a015cc3c640 100644 --- a/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/pipeline/buildFir.kt +++ b/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/pipeline/buildFir.kt @@ -5,7 +5,7 @@ package org.jetbrains.kotlin.fir.pipeline -import com.intellij.openapi.util.text.StringUtilRt +import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.diagnostics.DiagnosticReporter import org.jetbrains.kotlin.fir.FirSession import org.jetbrains.kotlin.fir.builder.PsiHandlingMode @@ -14,15 +14,12 @@ import org.jetbrains.kotlin.fir.declarations.FirFile import org.jetbrains.kotlin.fir.lightTree.LightTree2Fir import org.jetbrains.kotlin.fir.resolve.providers.firProvider import org.jetbrains.kotlin.fir.resolve.providers.impl.FirProviderImpl -import org.jetbrains.kotlin.fir.session.environment.AbstractProjectEnvironment import org.jetbrains.kotlin.fir.session.sourcesToPathsMapper import org.jetbrains.kotlin.psi.KtFile -import java.io.File -import java.io.FileNotFoundException +import org.jetbrains.kotlin.readSourceFileWithMapping fun FirSession.buildFirViaLightTree( - files: Collection, - projectEnvironment: AbstractProjectEnvironment, + files: Collection, diagnosticsReporter: DiagnosticReporter? = null, reportFilesAndLines: ((Int, Int) -> Unit)? = null ): List { @@ -32,14 +29,15 @@ fun FirSession.buildFirViaLightTree( val shouldCountLines = (reportFilesAndLines != null) var linesCount = 0 val firFiles = files.map { file -> - val text = projectEnvironment.getFileText(file.absolutePath) ?: throw FileNotFoundException(file.path) - val code = StringUtilRt.convertLineSeparators(text) - if (shouldCountLines) { - linesCount += code.count { it == '\n' } // assuming converted line separators + val (code, linesMapping) = file.getContentsAsStream().reader(Charsets.UTF_8).use { + it.readSourceFileWithMapping() } - builder.buildFirFile(code, file.name, file.path).also { firFile -> + if (shouldCountLines) { + linesCount += linesMapping.lastOffset + } + builder.buildFirFile(code, file, linesMapping).also { firFile -> firProvider.recordFile(firFile) - sourcesToPathsMapper.registerFileSource(firFile.source!!, file.path) + sourcesToPathsMapper.registerFileSource(firFile.source!!, file.path ?: file.name) } } reportFilesAndLines?.invoke(files.count(), linesCount) diff --git a/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/session/environment/AbstractProjectEnvironment.kt b/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/session/environment/AbstractProjectEnvironment.kt index 5cc2be1e743..517e3817041 100644 --- a/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/session/environment/AbstractProjectEnvironment.kt +++ b/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/session/environment/AbstractProjectEnvironment.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.fir.session.environment +import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.fir.FirModuleData import org.jetbrains.kotlin.fir.FirSession import org.jetbrains.kotlin.fir.java.FirJavaFacade @@ -49,6 +50,8 @@ interface AbstractProjectEnvironment { fun getSearchScopeByIoFiles(files: Iterable, allowOutOfProjectRoots: Boolean = false): AbstractProjectFileSearchScope + fun getSearchScopeBySourceFiles(files: Iterable, allowOutOfProjectRoots: Boolean = false): AbstractProjectFileSearchScope + fun getSearchScopeByDirectories(directories: Iterable): AbstractProjectFileSearchScope fun getSearchScopeForProjectLibraries(): AbstractProjectFileSearchScope @@ -60,6 +63,4 @@ interface AbstractProjectEnvironment { baseModuleData: FirModuleData, fileSearchScope: AbstractProjectFileSearchScope ): FirJavaFacade - - fun getFileText(filePath: String): String? } diff --git a/compiler/fir/fir2ir/src/org/jetbrains/kotlin/fir/backend/Fir2IrConverter.kt b/compiler/fir/fir2ir/src/org/jetbrains/kotlin/fir/backend/Fir2IrConverter.kt index 1b4c38b3ffa..735f41496e8 100644 --- a/compiler/fir/fir2ir/src/org/jetbrains/kotlin/fir/backend/Fir2IrConverter.kt +++ b/compiler/fir/fir2ir/src/org/jetbrains/kotlin/fir/backend/Fir2IrConverter.kt @@ -5,6 +5,8 @@ package org.jetbrains.kotlin.fir.backend +import org.jetbrains.kotlin.KtPsiSourceFileLinesMapping +import org.jetbrains.kotlin.KtSourceFileLinesMappingFromLineStartOffsets import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext import org.jetbrains.kotlin.backend.common.ir.BuiltinSymbolsBase @@ -167,7 +169,18 @@ class Fir2IrConverter( private fun registerFileAndClasses(file: FirFile, moduleFragment: IrModuleFragment) { val fileEntry = when (file.origin) { FirDeclarationOrigin.Source -> - file.psi?.let { PsiIrFileEntry(it as KtFile) } ?: NaiveSourceBasedFileEntryImpl(file.path ?: file.name, intArrayOf(0)) + file.psi?.let { PsiIrFileEntry(it as KtFile) } + ?: when (val linesMapping = file.sourceFileLinesMapping) { + is KtSourceFileLinesMappingFromLineStartOffsets -> + NaiveSourceBasedFileEntryImpl( + file.sourceFile?.path ?: file.sourceFile?.name ?: file.name, + linesMapping.lineStartOffsets, + linesMapping.lastOffset + ) + is KtPsiSourceFileLinesMapping -> PsiIrFileEntry(linesMapping.psiFile) + else -> + NaiveSourceBasedFileEntryImpl(file.sourceFile?.path ?: file.sourceFile?.name ?: file.name) + } FirDeclarationOrigin.Synthetic -> NaiveSourceBasedFileEntryImpl(file.name) else -> error("Unsupported file origin: ${file.origin}") } diff --git a/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/LightTree2Fir.kt b/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/LightTree2Fir.kt index 98632052987..66c2795a666 100644 --- a/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/LightTree2Fir.kt +++ b/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/LightTree2Fir.kt @@ -7,9 +7,10 @@ package org.jetbrains.kotlin.fir.lightTree import com.intellij.lang.LighterASTNode import com.intellij.lang.impl.PsiBuilderFactoryImpl -import com.intellij.openapi.util.io.FileUtil -import com.intellij.openapi.vfs.CharsetToolkit import com.intellij.util.diff.FlyweightCapableTreeStructure +import org.jetbrains.kotlin.KtIoFileSourceFile +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtSourceFileLinesMapping import org.jetbrains.kotlin.config.LanguageVersionSettings import org.jetbrains.kotlin.diagnostics.DiagnosticContext import org.jetbrains.kotlin.diagnostics.DiagnosticReporter @@ -22,6 +23,7 @@ import org.jetbrains.kotlin.fir.scopes.FirScopeProvider import org.jetbrains.kotlin.lexer.KotlinLexer import org.jetbrains.kotlin.parsing.KotlinLightParser import org.jetbrains.kotlin.parsing.KotlinParserDefinition +import org.jetbrains.kotlin.readSourceFileWithMapping import java.io.File import java.nio.file.Path @@ -46,7 +48,7 @@ class LightTree2Fir( return builder.lightTree } - fun buildLightTree(code: String): FlyweightCapableTreeStructure { + fun buildLightTree(code: CharSequence): FlyweightCapableTreeStructure { val builder = PsiBuilderFactoryImpl().createBuilder(parserDefinition, makeLexer(), code) KotlinLightParser.parse(builder) return builder.lightTree @@ -58,25 +60,25 @@ class LightTree2Fir( } fun buildFirFile(file: File): FirFile { - val code = FileUtil.loadFile(file, CharsetToolkit.UTF8, true) - return buildFirFile(code, file.name, file.path) + val sourceFile = KtIoFileSourceFile(file) + val (code, linesMapping) = with(file.inputStream().reader(Charsets.UTF_8)) { + this.readSourceFileWithMapping() + } + return buildFirFile(code, sourceFile, linesMapping) } - fun buildFirFile(lightTreeFile: LightTreeFile): FirFile = with(lightTreeFile) { + fun buildFirFile( + lightTree: FlyweightCapableTreeStructure, + sourceFile: KtSourceFile, + linesMapping: KtSourceFileLinesMapping + ): FirFile = DeclarationsConverter( session, scopeProvider, lightTree, diagnosticsReporter = diagnosticsReporter, - diagnosticContext = makeDiagnosticContext(path) - ).convertFile(lightTree.root, fileName, path) - } + diagnosticContext = makeDiagnosticContext(sourceFile.path) + ).convertFile(lightTree.root, sourceFile, linesMapping) - fun buildFirFile(code: String, fileName: String, path: String?): FirFile { - val lightTree = buildLightTree(code) - - return DeclarationsConverter( - session, scopeProvider, lightTree, diagnosticsReporter = diagnosticsReporter, - diagnosticContext = makeDiagnosticContext(path) - ).convertFile(lightTree.root, fileName, path) - } + fun buildFirFile(code: CharSequence, sourceFile: KtSourceFile, linesMapping: KtSourceFileLinesMapping): FirFile = + buildFirFile(buildLightTree(code), sourceFile, linesMapping) private fun makeDiagnosticContext(path: String?) = if (diagnosticsReporter == null) null else object : DiagnosticContext { @@ -86,9 +88,3 @@ class LightTree2Fir( } } -data class LightTreeFile( - val lightTree: FlyweightCapableTreeStructure, - val fileName: String, - val path: String? -) - diff --git a/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/converter/DeclarationsConverter.kt b/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/converter/DeclarationsConverter.kt index 83b72c1998d..cd85b3c6c0f 100644 --- a/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/converter/DeclarationsConverter.kt +++ b/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/converter/DeclarationsConverter.kt @@ -88,7 +88,7 @@ class DeclarationsConverter( * [org.jetbrains.kotlin.parsing.KotlinParsing.parseFile] * [org.jetbrains.kotlin.parsing.KotlinParsing.parsePreamble] */ - fun convertFile(file: LighterASTNode, fileName: String = "", filePath: String?): FirFile { + fun convertFile(file: LighterASTNode, sourceFile: KtSourceFile, linesMapping: KtSourceFileLinesMapping): FirFile { if (file.tokenType != KT_FILE) { //TODO throw error throw Exception() @@ -123,8 +123,9 @@ class DeclarationsConverter( source = file.toFirSourceElement() origin = FirDeclarationOrigin.Source moduleData = baseModuleData - name = fileName - path = filePath + name = sourceFile.name + this.sourceFile = sourceFile + this.sourceFileLinesMapping = linesMapping this.packageDirective = packageDirective ?: buildPackageDirective { packageFqName = context.packageFqName } annotations += fileAnnotationList imports += importList diff --git a/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/TotalKotlinTest.kt b/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/TotalKotlinTest.kt index dd8fb5f625e..97ea184e309 100644 --- a/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/TotalKotlinTest.kt +++ b/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/TotalKotlinTest.kt @@ -10,11 +10,15 @@ import com.intellij.openapi.vfs.CharsetToolkit import com.intellij.psi.impl.DebugUtil import com.intellij.testFramework.TestDataPath import com.intellij.util.PathUtil +import org.jetbrains.kotlin.KtIoFileSourceFile +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtSourceFileLinesMapping import org.jetbrains.kotlin.fir.FirRenderer import org.jetbrains.kotlin.fir.builder.AbstractRawFirBuilderTestCase import org.jetbrains.kotlin.fir.builder.StubFirScopeProvider import org.jetbrains.kotlin.fir.session.FirSessionFactory import org.jetbrains.kotlin.psi.KtFile +import org.jetbrains.kotlin.readSourceFileWithMapping import org.jetbrains.kotlin.test.JUnit3RunnerWithInners import org.junit.runner.RunWith import java.io.File @@ -33,12 +37,15 @@ class TotalKotlinTest : AbstractRawFirBuilderTestCase() { } } - private fun generateFirFromLightTree(onlyLightTree: Boolean, converter: LightTree2Fir, text: String, fileName: String, filePath: String) { + private fun generateFirFromLightTree( + onlyLightTree: Boolean, converter: LightTree2Fir, + text: CharSequence, sourceFile: KtSourceFile, linesMapping: KtSourceFileLinesMapping + ) { if (onlyLightTree) { val lightTree = LightTree2Fir.buildLightTree(text) DebugUtil.lightTreeToString(lightTree, false) } else { - val firFile = converter.buildFirFile(text, fileName, filePath) + val firFile = converter.buildFirFile(text, sourceFile, linesMapping) StringBuilder().also { FirRenderer(it).visitFile(firFile) }.toString() } } @@ -57,9 +64,12 @@ class TotalKotlinTest : AbstractRawFirBuilderTestCase() { if (onlyLightTree) println("LightTree generation") else println("Fir from LightTree converter") println("BASE PATH: $path") path.walkTopDown { - val text = FileUtil.loadFile(it, CharsetToolkit.UTF8, true).trim() + val sourceFile = KtIoFileSourceFile(it) + val (code, linesMapping) = with(it.inputStream().reader(Charsets.UTF_8)) { + this.readSourceFileWithMapping() + } time += measureNanoTime { - generateFirFromLightTree(onlyLightTree, lightTreeConverter, text, it.name, it.path) + generateFirFromLightTree(onlyLightTree, lightTreeConverter, code, sourceFile, linesMapping) } counter++ diff --git a/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/compare/TreesCompareTest.kt b/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/compare/TreesCompareTest.kt index 36326246e9c..d80d2b86c15 100644 --- a/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/compare/TreesCompareTest.kt +++ b/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/compare/TreesCompareTest.kt @@ -10,6 +10,8 @@ import com.intellij.openapi.vfs.CharsetToolkit import com.intellij.testFramework.TestDataPath import com.intellij.util.PathUtil import junit.framework.TestCase +import org.jetbrains.kotlin.KtInMemoryTextSourceFile +import org.jetbrains.kotlin.KtIoFileSourceFile import org.jetbrains.kotlin.checkers.BaseDiagnosticsTest.Companion.DIAGNOSTIC_IN_TESTDATA_PATTERN import org.jetbrains.kotlin.fir.FirRenderer import org.jetbrains.kotlin.fir.builder.AbstractRawFirBuilderTestCase @@ -19,7 +21,9 @@ import org.jetbrains.kotlin.fir.lightTree.walkTopDown import org.jetbrains.kotlin.fir.lightTree.walkTopDownWithTestData import org.jetbrains.kotlin.fir.session.FirSessionFactory import org.jetbrains.kotlin.psi.* +import org.jetbrains.kotlin.readSourceFileWithMapping import org.jetbrains.kotlin.test.JUnit3RunnerWithInners +import org.jetbrains.kotlin.toSourceLinesMapping import org.junit.runner.RunWith import java.io.File @@ -61,15 +65,17 @@ class TreesCompareTest : AbstractRawFirBuilderTestCase() { diagnosticsReporter = null ) compareBase(System.getProperty("user.dir"), withTestData = false) { file -> - val text = FileUtil.loadFile(file, CharsetToolkit.UTF8, true).trim() + val (text, linesMapping) = with(file.inputStream().reader(Charsets.UTF_8)) { + this.readSourceFileWithMapping() + } //psi - val ktFile = createPsiFile(FileUtil.getNameWithoutExtension(PathUtil.getFileName(file.path)), text) as KtFile + val ktFile = createPsiFile(FileUtil.getNameWithoutExtension(PathUtil.getFileName(file.path)), text.toString().trim()) as KtFile val firFileFromPsi = ktFile.toFirFile() val treeFromPsi = StringBuilder().also { FirRenderer(it).visitFile(firFileFromPsi) }.toString() //light tree - val firFileFromLightTree = lightTreeConverter.buildFirFile(text, file.name, file.path) + val firFileFromLightTree = lightTreeConverter.buildFirFile(text, KtIoFileSourceFile(file), linesMapping) val treeFromLightTree = StringBuilder().also { FirRenderer(it).visitFile(firFileFromLightTree) }.toString() return@compareBase treeFromLightTree == treeFromPsi @@ -100,7 +106,12 @@ class TreesCompareTest : AbstractRawFirBuilderTestCase() { .replace("".toRegex(), "") //light tree - val firFileFromLightTree = lightTreeConverter.buildFirFile(text, file.name, file.path) + val firFileFromLightTree = + lightTreeConverter.buildFirFile( + text, + KtInMemoryTextSourceFile(file.name, file.path, text), + text.toSourceLinesMapping() + ) val treeFromLightTree = StringBuilder().also { FirRenderer(it).visitFile(firFileFromLightTree) }.toString() .replace("".toRegex(), "") diff --git a/compiler/fir/raw-fir/psi2fir/src/org/jetbrains/kotlin/fir/builder/RawFirBuilder.kt b/compiler/fir/raw-fir/psi2fir/src/org/jetbrains/kotlin/fir/builder/RawFirBuilder.kt index 93090e20674..e72441e527e 100644 --- a/compiler/fir/raw-fir/psi2fir/src/org/jetbrains/kotlin/fir/builder/RawFirBuilder.kt +++ b/compiler/fir/raw-fir/psi2fir/src/org/jetbrains/kotlin/fir/builder/RawFirBuilder.kt @@ -911,7 +911,8 @@ open class RawFirBuilder( moduleData = baseModuleData origin = FirDeclarationOrigin.Source name = file.name - path = file.virtualFile?.path + sourceFile = KtPsiSourceFile(file) + sourceFileLinesMapping = KtPsiSourceFileLinesMapping(file) packageDirective = buildPackageDirective { packageFqName = context.packageFqName source = file.packageDirective?.toKtPsiSourceElement() diff --git a/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/FirFile.kt b/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/FirFile.kt index 55c66a3fe3e..3c31c9861ff 100644 --- a/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/FirFile.kt +++ b/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/FirFile.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.fir.declarations import org.jetbrains.kotlin.KtSourceElement +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtSourceFileLinesMapping import org.jetbrains.kotlin.fir.FirElement import org.jetbrains.kotlin.fir.FirModuleData import org.jetbrains.kotlin.fir.FirPackageDirective @@ -29,7 +31,8 @@ abstract class FirFile : FirDeclaration() { abstract val imports: List abstract val declarations: List abstract val name: String - abstract val path: String? + abstract val sourceFile: KtSourceFile? + abstract val sourceFileLinesMapping: KtSourceFileLinesMapping? abstract override val symbol: FirFileSymbol override fun accept(visitor: FirVisitor, data: D): R = visitor.visitFile(this, data) diff --git a/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/builder/FirFileBuilder.kt b/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/builder/FirFileBuilder.kt index 05ebe41fb65..27da97e5438 100644 --- a/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/builder/FirFileBuilder.kt +++ b/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/builder/FirFileBuilder.kt @@ -7,6 +7,8 @@ package org.jetbrains.kotlin.fir.declarations.builder import kotlin.contracts.* import org.jetbrains.kotlin.KtSourceElement +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtSourceFileLinesMapping import org.jetbrains.kotlin.fir.FirModuleData import org.jetbrains.kotlin.fir.FirPackageDirective import org.jetbrains.kotlin.fir.builder.FirAnnotationContainerBuilder @@ -39,7 +41,8 @@ class FirFileBuilder : FirAnnotationContainerBuilder { val imports: MutableList = mutableListOf() val declarations: MutableList = mutableListOf() lateinit var name: String - var path: String? = null + var sourceFile: KtSourceFile? = null + var sourceFileLinesMapping: KtSourceFileLinesMapping? = null override fun build(): FirFile { return FirFileImpl( @@ -53,7 +56,8 @@ class FirFileBuilder : FirAnnotationContainerBuilder { imports, declarations, name, - path, + sourceFile, + sourceFileLinesMapping, ) } diff --git a/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/impl/FirFileImpl.kt b/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/impl/FirFileImpl.kt index a638f5daedb..059ebaee09e 100644 --- a/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/impl/FirFileImpl.kt +++ b/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/impl/FirFileImpl.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.fir.declarations.impl import org.jetbrains.kotlin.KtSourceElement +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtSourceFileLinesMapping import org.jetbrains.kotlin.fir.FirModuleData import org.jetbrains.kotlin.fir.FirPackageDirective import org.jetbrains.kotlin.fir.declarations.FirDeclaration @@ -35,7 +37,8 @@ internal class FirFileImpl( override val imports: MutableList, override val declarations: MutableList, override val name: String, - override val path: String?, + override val sourceFile: KtSourceFile?, + override val sourceFileLinesMapping: KtSourceFileLinesMapping?, ) : FirFile() { override val symbol: FirFileSymbol = FirFileSymbol() diff --git a/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/NodeConfigurator.kt b/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/NodeConfigurator.kt index 17d726e7f3b..fe991e7a295 100644 --- a/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/NodeConfigurator.kt +++ b/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/NodeConfigurator.kt @@ -415,7 +415,8 @@ object NodeConfigurator : AbstractFieldConfigurator(FirTreeBuild +fieldList(import).withTransform() +declarations.withTransform() +stringField("name") - +stringField("path", nullable = true) + +field("sourceFile", sourceFileType, nullable = true) + +field("sourceFileLinesMapping", sourceFileLinesMappingType, nullable = true) +symbol("FirFileSymbol") } diff --git a/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/Types.kt b/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/Types.kt index 97b69d4093a..f3c78760229 100644 --- a/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/Types.kt +++ b/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/Types.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.fir.tree.generator import org.jetbrains.kotlin.KtSourceElement +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtSourceFileLinesMapping import org.jetbrains.kotlin.contracts.description.EventOccurrencesRange import org.jetbrains.kotlin.descriptors.ClassKind import org.jetbrains.kotlin.descriptors.Modality @@ -23,6 +25,8 @@ import org.jetbrains.kotlin.types.SmartcastStability import org.jetbrains.kotlin.types.Variance val sourceElementType = type(KtSourceElement::class) +val sourceFileType = type(KtSourceFile::class) +val sourceFileLinesMappingType = type(KtSourceFileLinesMapping::class) val jumpTargetType = type("fir", "FirTarget") val constKindType = type("types", "ConstantValueKind") val operationType = type("fir.expressions", "FirOperation") diff --git a/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceElement.kt b/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceElement.kt index 9178d1a7034..b3580609528 100644 --- a/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceElement.kt +++ b/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceElement.kt @@ -14,9 +14,6 @@ import com.intellij.psi.PsiFile import com.intellij.psi.PsiWhiteSpace import com.intellij.psi.tree.IElementType import com.intellij.util.diff.FlyweightCapableTreeStructure -import java.io.Closeable -import java.io.File -import java.io.InputStreamReader sealed class KtSourceElementKind @@ -444,90 +441,3 @@ inline fun LighterASTNode.toKtLightSourceElement( endOffset: Int = this.endOffset ): KtLightSourceElement = KtLightSourceElement(this, startOffset, endOffset, tree, kind) - -class KtSourceFilePosition(val line: Int, val column: Int, val lineContent: String?) { - - // NOTE: This method is used for presenting positions to the user - override fun toString(): String = if (line < 0) "(offset: $column line unknown)" else "($line,$column)" - - companion object { - val NONE = KtSourceFilePosition(-1, -1, null) - } -} - -class SequentialFilePositionFinder(file: File) : Closeable { - - private var reader: InputStreamReader = file.reader(/* TODO: select proper charset */) - - private var currentLineContent: String? = null - private val buffer = CharArray(255) - private var bufLength = -1 - private var bufPos = 0 - private var endOfStream = false - private var skipNextLf = false - - private var charsRead = 0 - private var currentLine = 0 - - // assuming that if called multiple times, calls should be sorted by ascending offset - fun findNextPosition(offset: Int, withLineContents: Boolean = true): KtSourceFilePosition { - assert(offset >= charsRead - (currentLineContent?.length ?: 0)) - - fun posInCurrentLine(): KtSourceFilePosition? { - val col = offset - (charsRead - currentLineContent!!.length - 1)/* beginning of line offset */ + 1 /* col is 1-based */ - return if (col <= currentLineContent!!.length) - KtSourceFilePosition(currentLine, col, if (withLineContents) currentLineContent else null) - else null - } - - if (offset < charsRead) { - return posInCurrentLine()!! - } - - while (true) { - if (currentLineContent == null) { - currentLineContent = readNextLine() - } - - posInCurrentLine()?.let { return@findNextPosition it } - - if (endOfStream) return KtSourceFilePosition(-1, offset, if (withLineContents) currentLineContent else null) - - currentLineContent = null - } - } - - private fun readNextLine() = buildString { - while (true) { - if (bufPos >= bufLength) { - bufLength = reader.read(buffer) - bufPos = 0 - if (bufLength < 0) { - endOfStream = true - break - } - } else { - val c = buffer[bufPos++] - charsRead++ - when { - c == '\n' && skipNextLf -> { - skipNextLf = false - } - c == '\n' || c == '\r' -> { - currentLine++ - skipNextLf = c == '\r' - break - } - else -> { - append(c) - skipNextLf = false - } - } - } - } - } - - override fun close() { - reader.close() - } -} diff --git a/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceFile.kt b/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceFile.kt new file mode 100644 index 00000000000..d481e64c594 --- /dev/null +++ b/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceFile.kt @@ -0,0 +1,57 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin + +import com.intellij.openapi.util.io.FileUtilRt +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.psi.PsiFile +import java.io.ByteArrayInputStream +import java.io.File +import java.io.InputStream + +interface KtSourceFile { + val name: String + val path: String? + + fun getContentsAsStream(): InputStream +} + +class KtPsiSourceFile(val psiFile: PsiFile) : KtSourceFile { + override val name: String + get() = psiFile.name + + override val path: String? + get() = psiFile.virtualFile?.path + + override fun getContentsAsStream(): InputStream = psiFile.virtualFile.inputStream +} + +class KtVirtualFileSourceFile(val virtualFile: VirtualFile) : KtSourceFile { + override val name: String + get() = virtualFile.name + + override val path: String + get() = virtualFile.path + + override fun getContentsAsStream(): InputStream = virtualFile.inputStream +} + +class KtIoFileSourceFile(val file: File) : KtSourceFile { + override val name: String + get() = file.name + override val path: String + get() = FileUtilRt.toSystemIndependentName(file.path) + + override fun getContentsAsStream(): InputStream = file.inputStream() +} + +class KtInMemoryTextSourceFile( + override val name: String, + override val path: String?, + val text: CharSequence +) : KtSourceFile { + override fun getContentsAsStream(): InputStream = ByteArrayInputStream(text.toString().toByteArray()) +} diff --git a/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceFileLinesMapping.kt b/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceFileLinesMapping.kt new file mode 100644 index 00000000000..5630d9c08a5 --- /dev/null +++ b/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceFileLinesMapping.kt @@ -0,0 +1,126 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin + +import com.intellij.openapi.editor.Document +import com.intellij.psi.PsiFile +import java.io.InputStreamReader + +interface KtSourceFileLinesMapping { + fun getLineStartOffset(line: Int): Int + fun getLineAndColumnByOffset(offset: Int): Pair + fun getLineByOffset(offset: Int): Int + + val lastOffset: Int + val linesCount: Int +} + +class KtPsiSourceFileLinesMapping(val psiFile: PsiFile) : KtSourceFileLinesMapping { + private val document: Document? by lazy { psiFile.viewProvider.document } + + override fun getLineStartOffset(line: Int): Int = + document?.getLineStartOffset(line) ?: -1 + + override fun getLineAndColumnByOffset(offset: Int): Pair = + document?.let { + val lineNumber = it.getLineNumber(offset) + val lineStartOffset = it.getLineStartOffset(lineNumber) + lineNumber to offset - lineStartOffset + } ?: (-1 to -1) + + override fun getLineByOffset(offset: Int): Int = + document?.getLineNumber(offset) ?: -1 + + override val lastOffset: Int + get() = document?.textLength ?: -1 + + override val linesCount: Int + get() = document?.lineCount ?: 0 +} + +open class KtSourceFileLinesMappingFromLineStartOffsets( + val lineStartOffsets: IntArray, override val lastOffset: Int +) : KtSourceFileLinesMapping { + override fun getLineStartOffset(line: Int): Int = lineStartOffsets[line] + + override fun getLineAndColumnByOffset(offset: Int): Pair { + val lineNumber = getLineByOffset(offset) + if (lineNumber < 0) return -1 to -1 + val lineStartOffset = lineStartOffsets[lineNumber] + return lineNumber to offset - lineStartOffset + } + + override fun getLineByOffset(offset: Int): Int { + val index = lineStartOffsets.binarySearch(offset) + return if (index >= 0) index else -index - 2 + } + + override val linesCount: Int + get() = lineStartOffsets.size +} + +/** + * Reads file contents from reader, converts line separators and calculates source lines to file offsets mapping + * + * Returns KtSourceFileLinesMapping and char sequence (StringBuilder to avoid premature copying) containing converted text + * The separators are converted similarly to the com.intellij.openapi.util.text.StringUtilRt algorithms + */ +fun InputStreamReader.readSourceFileWithMapping(): Pair { + val buffer = CharArray(255) + var bufLength = -1 + var bufPos = 0 + var skipNextLf = false + + var charsRead = 0 + + val lineOffsets = mutableListOf(0) // TODO: consider using implicit first line offset (needs to be handled properly in IR) + val sb = StringBuilder() + + while (true) { + if (bufPos >= bufLength) { + bufLength = read(buffer) + bufPos = 0 + if (bufLength < 0) { + break + } + } else { + val c = buffer[bufPos++] + charsRead++ + when { + c == '\n' && skipNextLf -> { + lineOffsets[lineOffsets.size - 1] = charsRead + skipNextLf = false + } + c == '\n' || c == '\r' -> { + sb.append('\n') + lineOffsets.add(charsRead) + skipNextLf = c == '\r' + } + else -> { + sb.append(c) + skipNextLf = false + } + } + } + } + + return sb to KtSourceFileLinesMappingFromLineStartOffsets(lineOffsets.toIntArray(), charsRead) +} + +/** + * Extracts source lines to offsets mapping from text + * + * intended for using mainly in tests, so no care is taken about performance or possible corner cases + */ +fun CharSequence.toSourceLinesMapping(): KtSourceFileLinesMapping { + val lineOffsets = mutableListOf(0) + var offset = 0 + for (c in this) { + offset++ + if (c == '\n') lineOffsets.add(offset) + } + return KtSourceFileLinesMappingFromLineStartOffsets(lineOffsets.toIntArray(), offset) +} diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt index 71c8c3174ea..ca46740b2bb 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt @@ -8,6 +8,9 @@ package org.jetbrains.kotlin.incremental import com.intellij.ide.highlighter.JavaFileType import com.intellij.openapi.util.Disposer import com.intellij.psi.PsiJavaModule +import org.jetbrains.kotlin.KtIoFileSourceFile +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtVirtualFileSourceFile import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension import org.jetbrains.kotlin.backend.jvm.JvmGeneratorExtensionsImpl import org.jetbrains.kotlin.backend.jvm.serialization.JvmIdSignatureDescriptor @@ -99,7 +102,7 @@ class IncrementalFirJvmCompilerRunner( val moduleName = args.moduleName ?: JvmProtoBufUtil.DEFAULT_MODULE_NAME val targetId = TargetId(moduleName, "java-production") // TODO: get rid of magic constant - val dirtySources = linkedSetOf().apply { addAll(sourcesToCompile) } + val dirtySources = linkedSetOf().apply { sourcesToCompile.forEach { add(KtIoFileSourceFile(it)) } } // TODO: probably shoudl be passed along with sourcesToCompile // TODO: file path normalization @@ -175,12 +178,11 @@ class IncrementalFirJvmCompilerRunner( createProjectEnvironment(configuration, rootDisposable, EnvironmentConfigFiles.JVM_CONFIG_FILES, messageCollector) // -sources - val allPlatformSourceFiles = linkedSetOf() // TODO: get from caller - val allCommonSourceFiles = linkedSetOf() + val allPlatformSourceFiles = linkedSetOf() // TODO: get from caller + val allCommonSourceFiles = linkedSetOf() configuration.kotlinSourceRoots.forAllFiles(configuration, projectEnvironment.project) { virtualFile, isCommon -> - val file = File(virtualFile.canonicalPath ?: virtualFile.path) - if (!file.isFile) error("TODO: better error: file not found $virtualFile") + val file = KtVirtualFileSourceFile(virtualFile) if (isCommon) allCommonSourceFiles.add(file) else allPlatformSourceFiles.add(file) } @@ -203,8 +205,8 @@ class IncrementalFirJvmCompilerRunner( val compilerInput = ModuleCompilerInput( targetId, - CommonPlatforms.defaultCommonPlatform, dirtySources.filter { it in allCommonSourceFiles }, - JvmPlatforms.unspecifiedJvmPlatform, dirtySources.filter { it in allPlatformSourceFiles }, + CommonPlatforms.defaultCommonPlatform, allCommonSourceFiles.filter { dirtySources.any { df -> df.path == it.path } }, + JvmPlatforms.unspecifiedJvmPlatform, allPlatformSourceFiles.filter { dirtySources.any { df -> df.path == it.path } }, configuration ) @@ -227,7 +229,10 @@ class IncrementalFirJvmCompilerRunner( mainClassFqName = findMainClass(analysisResults.fir) } - allCompiledSources.addAll(dirtySources) + // TODO: switch the whole IC to KtSourceFile instead of FIle + dirtySources.forEach { + allCompiledSources.add(File(it.path!!)) + } if (diagnosticsReporter.hasErrors) { diagnosticsReporter.reportToMessageCollector(messageCollector, renderDiagnosticName) @@ -250,7 +255,9 @@ class IncrementalFirJvmCompilerRunner( } } caches.inputsCache.removeOutputForSourceFiles(newDirtySources) - dirtySources.addAll(newDirtySources) + newDirtySources.forEach { + dirtySources.add(KtIoFileSourceFile(it)) + } projectEnvironment.localFileSystem.refresh(false) } } diff --git a/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/Utils.kt b/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/Utils.kt index 902146b4cde..2e1c6a8bba8 100644 --- a/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/Utils.kt +++ b/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/Utils.kt @@ -18,14 +18,9 @@ package org.jetbrains.kotlin.backend.common import org.jetbrains.kotlin.AbstractKtSourceElement import org.jetbrains.kotlin.KtOffsetsOnlySourceElement -import org.jetbrains.kotlin.KtRealPsiSourceElement -import org.jetbrains.kotlin.KtSourceElement -import org.jetbrains.kotlin.backend.common.psi.PsiSourceManager import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET -import org.jetbrains.kotlin.ir.declarations.IrDeclaration import org.jetbrains.kotlin.ir.declarations.IrFile -import org.jetbrains.kotlin.ir.util.render fun CommonBackendContext.reportWarning(message: String, irFile: IrFile?, irElement: IrElement) { report(irElement, irFile, message, false) @@ -37,12 +32,6 @@ fun MutableList.pop() = this.removeAt(size - 1) fun MutableList.peek(): E? = if (size == 0) null else this[size - 1] -fun findKtSourceElement(irElement: IrElement, irDeclaration: IrDeclaration): KtSourceElement { - val psiElement = PsiSourceManager.findPsiElement(irElement, irDeclaration) - ?: throw AssertionError("No PsiElement found for '${irElement.render()}'") - return KtRealPsiSourceElement(psiElement) -} - fun IrElement.sourceElement(): AbstractKtSourceElement? = if (startOffset != UNDEFINED_OFFSET) KtOffsetsOnlySourceElement(this.startOffset, this.endOffset) else null diff --git a/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/ir/JvmIrUtils.kt b/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/ir/JvmIrUtils.kt index 66118e6572a..93caf75b6ff 100644 --- a/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/ir/JvmIrUtils.kt +++ b/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/ir/JvmIrUtils.kt @@ -285,7 +285,7 @@ fun IrFile.getKtFile(): KtFile? = fun IrFile.getIoFile(): File? = when (val fe = fileEntry) { is PsiIrFileEntry -> fe.psiFile.virtualFile?.path?.let(::File) - else -> File(fe.name).takeIf { it.exists() } + else -> File(fe.name) } inline fun IrElement.hasChild(crossinline block: (IrElement) -> Boolean): Boolean { diff --git a/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/util/AdditionalIrUtils.kt b/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/util/AdditionalIrUtils.kt index 9a16228659a..69edeaf473b 100644 --- a/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/util/AdditionalIrUtils.kt +++ b/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/util/AdditionalIrUtils.kt @@ -186,7 +186,8 @@ val IrFileEntry.lineStartOffsets: IntArray class NaiveSourceBasedFileEntryImpl( override val name: String, - private val lineStartOffsets: IntArray = intArrayOf() + private val lineStartOffsets: IntArray = intArrayOf(), + override val maxOffset: Int = UNDEFINED_OFFSET ) : IrFileEntry { val lineStartOffsetsAreEmpty: Boolean get() = lineStartOffsets.isEmpty() @@ -215,15 +216,19 @@ class NaiveSourceBasedFileEntryImpl( if (offset == SYNTHETIC_OFFSET) return 0 if (offset < 0) return -1 val lineNumber = getLineNumber(offset) - return offset - lineStartOffsets[lineNumber] + return if (lineNumber < 0) -1 else offset - lineStartOffsets[lineNumber] } - override val maxOffset: Int - get() = UNDEFINED_OFFSET - - override fun getSourceRangeInfo(beginOffset: Int, endOffset: Int): SourceRangeInfo { - return SourceRangeInfo(name, beginOffset, -1, -1, endOffset, -1, -1) - } + override fun getSourceRangeInfo(beginOffset: Int, endOffset: Int): SourceRangeInfo = + SourceRangeInfo( + filePath = name, + startOffset = beginOffset, + startLineNumber = getLineNumber(beginOffset), + startColumnNumber = getColumnNumber(beginOffset), + endOffset = endOffset, + endLineNumber = getLineNumber(endOffset), + endColumnNumber = getColumnNumber(endOffset) + ) } private fun IrClass.getPropertyDeclaration(name: String): IrProperty? { diff --git a/compiler/test-infrastructure-utils/tests/org/jetbrains/kotlin/sourceFiles/LightTreeFile.kt b/compiler/test-infrastructure-utils/tests/org/jetbrains/kotlin/sourceFiles/LightTreeFile.kt new file mode 100644 index 00000000000..c0b4656ccbf --- /dev/null +++ b/compiler/test-infrastructure-utils/tests/org/jetbrains/kotlin/sourceFiles/LightTreeFile.kt @@ -0,0 +1,17 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.sourceFiles + +import com.intellij.lang.LighterASTNode +import com.intellij.util.diff.FlyweightCapableTreeStructure +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtSourceFileLinesMapping + +data class LightTreeFile( + val lightTree: FlyweightCapableTreeStructure, + val sourceFile: KtSourceFile, + val linesMapping: KtSourceFileLinesMapping +) diff --git a/compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/SourceFileProvider.kt b/compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/SourceFileProvider.kt index edfeadab85b..41c1f932814 100644 --- a/compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/SourceFileProvider.kt +++ b/compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/SourceFileProvider.kt @@ -6,11 +6,13 @@ package org.jetbrains.kotlin.test.services import com.intellij.openapi.project.Project +import org.jetbrains.kotlin.KtInMemoryTextSourceFile import org.jetbrains.kotlin.fir.lightTree.LightTree2Fir -import org.jetbrains.kotlin.fir.lightTree.LightTreeFile import org.jetbrains.kotlin.psi.KtFile +import org.jetbrains.kotlin.sourceFiles.LightTreeFile import org.jetbrains.kotlin.test.model.TestFile import org.jetbrains.kotlin.test.util.KtTestUtil +import org.jetbrains.kotlin.toSourceLinesMapping import java.io.File abstract class SourceFilePreprocessor(val testServices: TestServices) { @@ -101,9 +103,10 @@ fun SourceFileProvider.getKtFilesForSourceFiles(testFiles: Collection, fun SourceFileProvider.getLightTreeKtFileForSourceFile(testFile: TestFile): LightTreeFile { val shortName = testFile.name.substringAfterLast('/').substringAfterLast('\\') - val file = getRealFileForSourceFile(testFile) - val lightTree = LightTree2Fir.buildLightTree(file.readText()) - return LightTreeFile(lightTree, shortName, "/$shortName") // emulating behavior of KtTestUtil.createFile so path looks the same in testdata + val sourceFile = KtInMemoryTextSourceFile(shortName, "/$shortName", getContentOfSourceFile(testFile)) + val linesMapping = sourceFile.text.toSourceLinesMapping() + val lightTree = LightTree2Fir.buildLightTree(sourceFile.text) + return LightTreeFile(lightTree, sourceFile, linesMapping) } fun SourceFileProvider.getLightTreeFilesForSourceFiles(testFiles: Collection): Map { diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/classic/ClassicJvmBackendFacade.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/classic/ClassicJvmBackendFacade.kt index aa650d468d6..9748a41c6c0 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/classic/ClassicJvmBackendFacade.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/classic/ClassicJvmBackendFacade.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.test.backend.classic +import org.jetbrains.kotlin.KtPsiSourceFile import org.jetbrains.kotlin.codegen.ClassBuilderFactories import org.jetbrains.kotlin.codegen.DefaultCodegenFactory import org.jetbrains.kotlin.codegen.KotlinCodegenFacade @@ -46,7 +47,7 @@ class ClassicJvmBackendFacade( javaCompilerFacade.compileJavaFiles(module, configuration, generationState.factory) return BinaryArtifacts.Jvm( generationState.factory, - psiFiles.map { SourceFileInfo(it, JvmFileClassUtil.getFileClassInfoNoResolve(it)) } + psiFiles.map { SourceFileInfo(KtPsiSourceFile(it), JvmFileClassUtil.getFileClassInfoNoResolve(it)) } ) } } diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/IrTextDumpHandler.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/IrTextDumpHandler.kt index 1d822b59854..70256c3a85d 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/IrTextDumpHandler.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/IrTextDumpHandler.kt @@ -14,7 +14,10 @@ import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerDesc import org.jetbrains.kotlin.ir.declarations.IrClass import org.jetbrains.kotlin.ir.declarations.IrFile import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl -import org.jetbrains.kotlin.ir.util.* +import org.jetbrains.kotlin.ir.util.DeclarationStubGenerator +import org.jetbrains.kotlin.ir.util.SymbolTable +import org.jetbrains.kotlin.ir.util.dump +import org.jetbrains.kotlin.ir.util.dumpTreesFromLineNumber import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.psi2ir.generators.DeclarationStubGeneratorImpl @@ -46,7 +49,7 @@ class IrTextDumpHandler(testServices: TestServices) : AbstractIrHandler(testServ } fun List.groupWithTestFiles(module: TestModule): List> = mapNotNull { irFile -> - val name = irFile.fileEntry.name.removePrefix("/") + val name = File(irFile.fileEntry.name).name val testFile = module.files.firstOrNull { it.name == name } testFile to irFile } diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/JvmBoxRunner.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/JvmBoxRunner.kt index 66c266dc983..1682f819f13 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/JvmBoxRunner.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/JvmBoxRunner.kt @@ -61,7 +61,7 @@ open class JvmBoxRunner(testServices: TestServices) : JvmBinaryArtifactHandler(t val classLoader = createAndVerifyClassLoader(module, info.classFileFactory, reportProblems) try { for (fileInfo in fileInfos) { - if (fileContainsBoxMethod(fileInfo.file)) { + if (fileContainsBoxMethod(fileInfo.sourceFile)) { boxMethodFound = true callBoxMethodAndCheckResultWithCleanup( fileInfo.info, diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/SMAPDumpHandler.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/SMAPDumpHandler.kt index b330f91de84..7072d1869e1 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/SMAPDumpHandler.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/SMAPDumpHandler.kt @@ -19,6 +19,7 @@ import org.jetbrains.kotlin.test.services.TestServices import org.jetbrains.kotlin.test.services.moduleStructure import org.jetbrains.kotlin.test.utils.MultiModuleInfoDumper import org.jetbrains.kotlin.test.utils.withExtension +import java.io.File class SMAPDumpHandler(testServices: TestServices) : JvmBinaryArtifactHandler(testServices) { companion object { @@ -39,7 +40,7 @@ class SMAPDumpHandler(testServices: TestServices) : JvmBinaryArtifactHandler(tes val originalFileNames = module.files.map { it.name } val compiledSmaps = CommonSMAPTestUtil.extractSMAPFromClasses(info.classFileFactory.getClassFiles()).mapNotNull { - val name = it.sourceFile.removePrefix("/") + val name = File(it.sourceFile).name val index = originalFileNames.indexOf(name) val testFile = module.files[index] if (NO_SMAP_DUMP in testFile.directives) return@mapNotNull null @@ -57,7 +58,7 @@ class SMAPDumpHandler(testServices: TestServices) : JvmBinaryArtifactHandler(tes dumper.builderForModule(module).apply { for (source in compiledData.values) { - appendLine("// FILE: ${source.sourceFile.removePrefix("/")}") + appendLine("// FILE: ${File(source.sourceFile).name}") appendLine(source.smap ?: "") } } diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt index eca80223238..2da82e565d0 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.test.backend.ir +import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.backend.jvm.JvmIrCodegenFactory import org.jetbrains.kotlin.codegen.CodegenFactory import org.jetbrains.kotlin.codegen.state.GenerationState @@ -35,7 +36,8 @@ sealed class IrBackendInput : ResultingArtifact.BackendInput() { data class JvmIrBackendInput( val state: GenerationState, val codegenFactory: JvmIrCodegenFactory, - val backendInput: JvmIrCodegenFactory.JvmIrBackendInput + val backendInput: JvmIrCodegenFactory.JvmIrBackendInput, + val sourceFiles: List ) : IrBackendInput() { override val irModuleFragment: IrModuleFragment get() = backendInput.irModuleFragment diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/JvmIrBackendFacade.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/JvmIrBackendFacade.kt index ecb79ba0989..a81e79e5334 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/JvmIrBackendFacade.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/JvmIrBackendFacade.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.test.backend.ir +import org.jetbrains.kotlin.KtPsiSourceFile import org.jetbrains.kotlin.backend.common.BackendException import org.jetbrains.kotlin.backend.jvm.MultifileFacadeFileEntry import org.jetbrains.kotlin.backend.jvm.lower.getFileClassInfoFromIrFile @@ -21,7 +22,6 @@ import org.jetbrains.kotlin.test.model.SourceFileInfo import org.jetbrains.kotlin.test.model.TestModule import org.jetbrains.kotlin.test.services.TestServices import org.jetbrains.kotlin.test.services.compilerConfigurationProvider -import java.io.File class JvmIrBackendFacade( testServices: TestServices @@ -48,14 +48,20 @@ class JvmIrBackendFacade( val configuration = testServices.compilerConfigurationProvider.getCompilerConfiguration(module) javaCompilerFacade.compileJavaFiles(module, configuration, state.factory) - fun sourceFileInfos(irFile: IrFile, allowNestedMultifileFacades: Boolean): List> = + fun sourceFileInfos(irFile: IrFile, allowNestedMultifileFacades: Boolean): List = when (val fileEntry = irFile.fileEntry) { is PsiIrFileEntry -> { - listOf(SourceFileInfo(fileEntry.psiFile, JvmFileClassUtil.getFileClassInfoNoResolve(fileEntry.psiFile as KtFile))) + listOf( + SourceFileInfo( + KtPsiSourceFile(fileEntry.psiFile), + JvmFileClassUtil.getFileClassInfoNoResolve(fileEntry.psiFile as KtFile) + ) + ) } is NaiveSourceBasedFileEntryImpl -> { - val file = File(fileEntry.name) - listOf(SourceFileInfo(file, getFileClassInfoFromIrFile(irFile, file.name))) + val sourceFile = inputArtifact.sourceFiles.find { it.path == fileEntry.name } + if (sourceFile == null) emptyList() // synthetic files, like CoroutineHelpers.kt, are ignored here + else listOf(SourceFileInfo(sourceFile, getFileClassInfoFromIrFile(irFile, sourceFile.name))) } is MultifileFacadeFileEntry -> { if (!allowNestedMultifileFacades) error("nested multi-file facades are not allowed") diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontend2IrConverter.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontend2IrConverter.kt index 95edb453b3f..6821f0c83d6 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontend2IrConverter.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontend2IrConverter.kt @@ -64,7 +64,8 @@ class ClassicFrontend2IrConverter( return IrBackendInput.JvmIrBackendInput( state, codegenFactory, - codegenFactory.convertToIr(CodegenFactory.IrConversionInput.fromGenerationStateAndFiles(state, psiFiles.values)) + codegenFactory.convertToIr(CodegenFactory.IrConversionInput.fromGenerationStateAndFiles(state, psiFiles.values)), + emptyList() ) } diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/Fir2IrResultsConverter.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/Fir2IrResultsConverter.kt index e00e416f65e..2c6d7a8d344 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/Fir2IrResultsConverter.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/Fir2IrResultsConverter.kt @@ -50,6 +50,7 @@ class Fir2IrResultsConverter( // TODO: handle fir from light tree val ktFiles = inputArtifact.firFiles.values.mapNotNull { it.psi as KtFile? } + val sourceFiles = inputArtifact.firFiles.values.mapNotNull { it.sourceFile } // Create and initialize the module and its dependencies val project = compilerConfigurationProvider.getProject(module) @@ -82,7 +83,8 @@ class Fir2IrResultsConverter( extensions, FirJvmBackendExtension(inputArtifact.session, components), notifyCodegenStart = {}, - ) + ), + sourceFiles ) } } diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFrontendFacade.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFrontendFacade.kt index aaedb253515..99d3e52da45 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFrontendFacade.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFrontendFacade.kt @@ -18,7 +18,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.VfsBasedProjectEnvironment import org.jetbrains.kotlin.cli.jvm.config.jvmClasspathRoots import org.jetbrains.kotlin.cli.jvm.config.jvmModularRoots import org.jetbrains.kotlin.config.JVMConfigurationKeys -import org.jetbrains.kotlin.fir.analysis.FirAnalyzerFacade +import org.jetbrains.kotlin.fir.FirAnalyzerFacade import org.jetbrains.kotlin.fir.checkers.registerExtendedCommonCheckers import org.jetbrains.kotlin.fir.extensions.FirExtensionRegistrar import org.jetbrains.kotlin.fir.moduleData diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirOutputArtifact.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirOutputArtifact.kt index 7ba52fe1c27..bb7d2fb87b7 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirOutputArtifact.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirOutputArtifact.kt @@ -5,9 +5,9 @@ package org.jetbrains.kotlin.test.frontend.fir +import org.jetbrains.kotlin.fir.AbstractFirAnalyzerFacade +import org.jetbrains.kotlin.fir.FirAnalyzerFacade import org.jetbrains.kotlin.fir.FirSession -import org.jetbrains.kotlin.fir.analysis.AbstractFirAnalyzerFacade -import org.jetbrains.kotlin.fir.analysis.FirAnalyzerFacade import org.jetbrains.kotlin.fir.declarations.FirFile import org.jetbrains.kotlin.test.model.FrontendKinds import org.jetbrains.kotlin.test.model.ResultingArtifact diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/model/ResultingArtifacts.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/model/ResultingArtifacts.kt index 3fc0ed18f43..ae8d30b52c8 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/model/ResultingArtifacts.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/model/ResultingArtifacts.kt @@ -5,19 +5,20 @@ package org.jetbrains.kotlin.test.model +import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.codegen.ClassFileFactory import org.jetbrains.kotlin.fileClasses.JvmFileClassInfo import org.jetbrains.kotlin.ir.backend.js.CompilerResult import org.jetbrains.kotlin.js.facade.TranslationResult import java.io.File -class SourceFileInfo( - val file: F, +class SourceFileInfo( + val sourceFile: KtSourceFile, val info: JvmFileClassInfo ) object BinaryArtifacts { - class Jvm(val classFileFactory: ClassFileFactory, val fileInfos: Collection>) : ResultingArtifact.Binary() { + class Jvm(val classFileFactory: ClassFileFactory, val fileInfos: Collection) : ResultingArtifact.Binary() { override val kind: BinaryKind get() = ArtifactKinds.Jvm } diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/services/sourceProviders/MainFunctionForBlackBoxTestsSourceProvider.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/services/sourceProviders/MainFunctionForBlackBoxTestsSourceProvider.kt index 4ef570b4a30..59d91ea7ea2 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/services/sourceProviders/MainFunctionForBlackBoxTestsSourceProvider.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/services/sourceProviders/MainFunctionForBlackBoxTestsSourceProvider.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.test.services.sourceProviders import com.intellij.psi.PsiFile +import org.jetbrains.kotlin.KtPsiSourceFile +import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.test.directives.CodegenTestDirectives.REQUIRES_SEPARATE_PROCESS import org.jetbrains.kotlin.test.directives.JvmEnvironmentConfigurationDirectives.JDK_KIND import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives @@ -39,11 +41,12 @@ open class MainFunctionForBlackBoxTestsSourceProvider(testServices: TestServices return containsBoxMethod(file.originalContent) } - fun fileContainsBoxMethod(file: T): Boolean = - when (file) { - is PsiFile -> containsBoxMethod(file.text) - is File -> containsBoxMethod(file.readText()) - else -> error("Unknown file type: $file") + fun fileContainsBoxMethod(sourceFile: KtSourceFile): Boolean = + when (sourceFile) { + is KtPsiSourceFile -> containsBoxMethod(sourceFile.psiFile.text) + else -> with(sourceFile.getContentsAsStream().reader(Charsets.UTF_8)) { + containsBoxMethod(this.readText()) + } } fun containsBoxMethod(fileContent: String): Boolean { diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/SMAPTestUtil.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/SMAPTestUtil.kt index 96c7594f489..ad601fb261e 100644 --- a/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/SMAPTestUtil.kt +++ b/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/SMAPTestUtil.kt @@ -25,6 +25,7 @@ import org.jetbrains.kotlin.codegen.inline.GENERATE_SMAP import org.jetbrains.kotlin.test.KotlinBaseTest import org.jetbrains.kotlin.test.util.JUnit4Assertions import org.junit.Assert +import java.io.File import java.io.StringReader object SMAPTestUtil { @@ -62,7 +63,7 @@ object SMAPTestUtil { }.associateBy { it.sourceFile } for (source in sourceData) { - val ktFileName = "/" + source.sourceFile + val ktFileName = "/" + File(source.sourceFile).name .replace(".smap-nonseparate-compilation", ".kt") .replace(".smap-separate-compilation", ".kt") .replace(".smap", ".kt") diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/fir/AbstractFirBaseDiagnosticsTest.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/fir/AbstractFirBaseDiagnosticsTest.kt index a64eb0651f8..2839725ea8c 100644 --- a/compiler/tests-common/tests/org/jetbrains/kotlin/fir/AbstractFirBaseDiagnosticsTest.kt +++ b/compiler/tests-common/tests/org/jetbrains/kotlin/fir/AbstractFirBaseDiagnosticsTest.kt @@ -11,6 +11,7 @@ import com.intellij.openapi.vfs.VirtualFileManager import com.intellij.psi.PsiElement import com.intellij.psi.PsiElementFinder import com.intellij.psi.search.GlobalSearchScope +import org.jetbrains.kotlin.KtInMemoryTextSourceFile import org.jetbrains.kotlin.analyzer.ModuleInfo import org.jetbrains.kotlin.asJava.finder.JavaElementFinder import org.jetbrains.kotlin.checkers.BaseDiagnosticsTest @@ -46,6 +47,7 @@ import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.resolve.AnalyzingUtils import org.jetbrains.kotlin.resolve.PlatformDependentAnalyzerServices import org.jetbrains.kotlin.resolve.jvm.platform.JvmPlatformAnalyzerServices +import org.jetbrains.kotlin.toSourceLinesMapping import java.io.File abstract class AbstractFirBaseDiagnosticsTest : BaseDiagnosticsTest() { @@ -133,7 +135,12 @@ abstract class AbstractFirBaseDiagnosticsTest : BaseDiagnosticsTest() { if (useLightTree) { val lightTreeBuilder = LightTree2Fir(session, firProvider.kotlinScopeProvider) ktFiles.mapTo(firFiles) { - val firFile = lightTreeBuilder.buildFirFile(it.text, it.name, it.virtualFilePath) + val firFile = + lightTreeBuilder.buildFirFile( + it.text, + KtInMemoryTextSourceFile(it.name, it.virtualFilePath, it.text), + it.text.toSourceLinesMapping() + ) (session.firProvider as FirProviderImpl).recordFile(firFile) firFile } diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/fir/FirResolveBench.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/fir/FirResolveBench.kt index 9f28f4a4dbb..790006c8e4f 100644 --- a/compiler/tests-common/tests/org/jetbrains/kotlin/fir/FirResolveBench.kt +++ b/compiler/tests-common/tests/org/jetbrains/kotlin/fir/FirResolveBench.kt @@ -9,6 +9,7 @@ import com.intellij.openapi.util.io.FileUtil import com.intellij.openapi.util.text.StringUtil import com.intellij.openapi.vfs.CharsetToolkit import com.intellij.psi.PsiElement +import org.jetbrains.kotlin.KtIoFileSourceFile import org.jetbrains.kotlin.fir.builder.RawFirBuilder import org.jetbrains.kotlin.fir.declarations.FirFile import org.jetbrains.kotlin.fir.diagnostics.ConeStubDiagnostic @@ -26,6 +27,7 @@ import org.jetbrains.kotlin.fir.types.* import org.jetbrains.kotlin.fir.visitors.FirDefaultVisitorVoid import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.psi.psiUtil.startOffset +import org.jetbrains.kotlin.readSourceFileWithMapping import org.jetbrains.kotlin.utils.addToStdlib.sumByLong import java.io.File import java.io.PrintStream @@ -139,16 +141,18 @@ class FirResolveBench(val withProgress: Boolean, val listener: BenchListener? = return files.map { file -> val before = vmStateSnapshot() val firFile: FirFile - val code: String val time = measureNanoTime { - code = FileUtil.loadFile(file, CharsetToolkit.UTF8, true).trim() - firFile = builder.buildFirFile(code, file.name, file.path) + val sourceFile = KtIoFileSourceFile(file) + val (code, linesMapping) = with(file.inputStream().reader(Charsets.UTF_8)) { + this.readSourceFileWithMapping() + } + totalLines += linesMapping.linesCount + firFile = builder.buildFirFile(code, sourceFile, linesMapping) (builder.session.firProvider as FirProviderImpl).recordFile(firFile) } val after = vmStateSnapshot() val diff = after - before recordTime(builder::class, diff, time) - totalLines += StringUtil.countNewLines(code) firFile }.also { listener?.after(builder::class) diff --git a/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt b/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt index ada3ea7902b..6349f634afc 100644 --- a/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt +++ b/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt @@ -29,7 +29,7 @@ import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.JVMConfigurationKeys import org.jetbrains.kotlin.config.languageVersionSettings -import org.jetbrains.kotlin.fir.analysis.FirAnalyzerFacade +import org.jetbrains.kotlin.fir.FirAnalyzerFacade import org.jetbrains.kotlin.fir.backend.jvm.FirJvmBackendClassResolver import org.jetbrains.kotlin.fir.backend.jvm.FirJvmBackendExtension import org.jetbrains.kotlin.fir.createSessionForTests diff --git a/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/analysis/FirAnalyzerFacade.kt b/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/fir/FirAnalyzerFacade.kt similarity index 94% rename from compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/analysis/FirAnalyzerFacade.kt rename to compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/fir/FirAnalyzerFacade.kt index b0642f5c40c..f9af763d3ec 100644 --- a/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/analysis/FirAnalyzerFacade.kt +++ b/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/fir/FirAnalyzerFacade.kt @@ -1,17 +1,16 @@ /* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.fir.analysis +package org.jetbrains.kotlin.fir import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension import org.jetbrains.kotlin.backend.jvm.serialization.JvmIdSignatureDescriptor import org.jetbrains.kotlin.config.LanguageVersionSettings -import org.jetbrains.kotlin.fir.FirSession -import org.jetbrains.kotlin.fir.analysis.collectors.FirDiagnosticsCollector import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory import org.jetbrains.kotlin.diagnostics.KtDiagnostic +import org.jetbrains.kotlin.fir.analysis.collectors.FirDiagnosticsCollector import org.jetbrains.kotlin.fir.backend.Fir2IrConverter import org.jetbrains.kotlin.fir.backend.Fir2IrResult import org.jetbrains.kotlin.fir.backend.jvm.Fir2IrJvmSpecialAnnotationSymbolProvider @@ -21,8 +20,6 @@ import org.jetbrains.kotlin.fir.builder.PsiHandlingMode import org.jetbrains.kotlin.fir.builder.RawFirBuilder import org.jetbrains.kotlin.fir.declarations.FirFile import org.jetbrains.kotlin.fir.lightTree.LightTree2Fir -import org.jetbrains.kotlin.fir.lightTree.LightTreeFile -import org.jetbrains.kotlin.fir.moduleData import org.jetbrains.kotlin.fir.resolve.ScopeSession import org.jetbrains.kotlin.fir.resolve.providers.firProvider import org.jetbrains.kotlin.fir.resolve.providers.impl.FirProviderImpl @@ -31,6 +28,7 @@ import org.jetbrains.kotlin.ir.backend.jvm.serialization.JvmDescriptorMangler import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.psi2ir.generators.GeneratorExtensions +import org.jetbrains.kotlin.sourceFiles.LightTreeFile abstract class AbstractFirAnalyzerFacade { abstract val scopeSession: ScopeSession @@ -63,7 +61,7 @@ class FirAnalyzerFacade( firFiles = if (useLightTree) { val builder = LightTree2Fir(session, firProvider.kotlinScopeProvider) lightTreeFiles.map { - builder.buildFirFile(it).also { firFile -> + builder.buildFirFile(it.lightTree, it.sourceFile, it.linesMapping).also { firFile -> firProvider.recordFile(firFile) } }