diff --git a/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/compiler/based/LowLevelFirAnalyzerFacade.kt b/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/compiler/based/LowLevelFirAnalyzerFacade.kt index 7590f1195a0..e8edb52e864 100644 --- a/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/compiler/based/LowLevelFirAnalyzerFacade.kt +++ b/analysis/low-level-api-fir/tests/org/jetbrains/kotlin/analysis/low/level/api/fir/compiler/based/LowLevelFirAnalyzerFacade.kt @@ -8,8 +8,8 @@ package org.jetbrains.kotlin.analysis.low.level.api.fir.compiler.based import org.jetbrains.kotlin.analysis.low.level.api.fir.api.DiagnosticCheckerFilter import org.jetbrains.kotlin.analysis.low.level.api.fir.api.LLFirModuleResolveState import org.jetbrains.kotlin.analysis.low.level.api.fir.api.collectDiagnosticsForFile -import org.jetbrains.kotlin.fir.analysis.AbstractFirAnalyzerFacade import org.jetbrains.kotlin.diagnostics.KtDiagnostic +import org.jetbrains.kotlin.fir.AbstractFirAnalyzerFacade import org.jetbrains.kotlin.fir.backend.Fir2IrResult import org.jetbrains.kotlin.fir.declarations.FirFile import org.jetbrains.kotlin.fir.declarations.FirResolvePhase diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/common/fir/FirDiagnosticsCompilerResultsReporter.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/common/fir/FirDiagnosticsCompilerResultsReporter.kt index 9827a107deb..f8e09bc01d6 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/common/fir/FirDiagnosticsCompilerResultsReporter.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/common/fir/FirDiagnosticsCompilerResultsReporter.kt @@ -5,7 +5,6 @@ package org.jetbrains.kotlin.cli.common.fir -import org.jetbrains.kotlin.SequentialFilePositionFinder import org.jetbrains.kotlin.cli.common.messages.* import org.jetbrains.kotlin.diagnostics.DiagnosticUtils import org.jetbrains.kotlin.diagnostics.KtDiagnostic @@ -13,7 +12,9 @@ import org.jetbrains.kotlin.diagnostics.KtPsiDiagnostic import org.jetbrains.kotlin.diagnostics.Severity import org.jetbrains.kotlin.diagnostics.impl.BaseDiagnosticsCollector import org.jetbrains.kotlin.diagnostics.rendering.RootDiagnosticRendererFactory +import java.io.Closeable import java.io.File +import java.io.InputStreamReader object FirDiagnosticsCompilerResultsReporter { fun reportToMessageCollector( @@ -55,6 +56,7 @@ object FirDiagnosticsCompilerResultsReporter { ) } else -> { + // TODO: bring KtSourceFile and KtSourceFileLinesMapping here and rewrite reporting via it to avoid code duplication // NOTE: SequentialPositionFinder relies on the ascending order of the input offsets, so the code relies // on the the appropriate sorting above // Also the end offset is ignored, as it is irrelevant for the CLI reporting @@ -132,4 +134,91 @@ object FirDiagnosticsCompilerResultsReporter { fun BaseDiagnosticsCollector.reportToMessageCollector(messageCollector: MessageCollector, renderDiagnosticName: Boolean) { FirDiagnosticsCompilerResultsReporter.reportToMessageCollector(this, messageCollector, renderDiagnosticName) -} \ No newline at end of file +} + +private class KtSourceFilePos(val line: Int, val column: Int, val lineContent: String?) { + + // NOTE: This method is used for presenting positions to the user + override fun toString(): String = if (line < 0) "(offset: $column line unknown)" else "($line,$column)" + + companion object { + val NONE = KtSourceFilePos(-1, -1, null) + } +} + +private class SequentialFilePositionFinder(file: File) : Closeable { + + private var reader: InputStreamReader = file.reader(/* TODO: select proper charset */) + + private var currentLineContent: String? = null + private val buffer = CharArray(255) + private var bufLength = -1 + private var bufPos = 0 + private var endOfStream = false + private var skipNextLf = false + + private var charsRead = 0 + private var currentLine = 0 + + // assuming that if called multiple times, calls should be sorted by ascending offset + fun findNextPosition(offset: Int, withLineContents: Boolean = true): KtSourceFilePos { + assert(offset >= charsRead - (currentLineContent?.length ?: 0)) + + fun posInCurrentLine(): KtSourceFilePos? { + val col = offset - (charsRead - currentLineContent!!.length - 1)/* beginning of line offset */ + 1 /* col is 1-based */ + return if (col <= currentLineContent!!.length) + KtSourceFilePos(currentLine, col, if (withLineContents) currentLineContent else null) + else null + } + + if (offset < charsRead) { + return posInCurrentLine()!! + } + + while (true) { + if (currentLineContent == null) { + currentLineContent = readNextLine() + } + + posInCurrentLine()?.let { return@findNextPosition it } + + if (endOfStream) return KtSourceFilePos(-1, offset, if (withLineContents) currentLineContent else null) + + currentLineContent = null + } + } + + private fun readNextLine() = buildString { + while (true) { + if (bufPos >= bufLength) { + bufLength = reader.read(buffer) + bufPos = 0 + if (bufLength < 0) { + endOfStream = true + break + } + } else { + val c = buffer[bufPos++] + charsRead++ + when { + c == '\n' && skipNextLf -> { + skipNextLf = false + } + c == '\n' || c == '\r' -> { + currentLine++ + skipNextLf = c == '\r' + break + } + else -> { + append(c) + skipNextLf = false + } + } + } + } + } + + override fun close() { + reader.close() + } +} diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/VfsBasedProjectEnvironment.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/VfsBasedProjectEnvironment.kt index d5532f37e67..0afedde160e 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/VfsBasedProjectEnvironment.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/VfsBasedProjectEnvironment.kt @@ -7,12 +7,17 @@ package org.jetbrains.kotlin.cli.jvm.compiler import com.intellij.openapi.project.Project import com.intellij.openapi.vfs.StandardFileSystems +import com.intellij.openapi.vfs.VirtualFile import com.intellij.openapi.vfs.VirtualFileManager import com.intellij.openapi.vfs.VirtualFileSystem import com.intellij.psi.PsiElementFinder import com.intellij.psi.PsiFile import com.intellij.psi.search.GlobalSearchScope import com.intellij.psi.search.ProjectScope +import org.jetbrains.kotlin.KtIoFileSourceFile +import org.jetbrains.kotlin.KtPsiSourceFile +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtVirtualFileSourceFile import org.jetbrains.kotlin.asJava.finder.JavaElementFinder import org.jetbrains.kotlin.fir.FirModuleData import org.jetbrains.kotlin.fir.FirSession @@ -65,16 +70,33 @@ open class VfsBasedProjectEnvironment( psiFinderExtensionPoint.registerExtension(FirJavaElementFinder(firSession, project), project) } + private fun List.toSearchScope(allowOutOfProjectRoots: Boolean) = + takeIf { it.isNotEmpty() } + ?.let { + if (allowOutOfProjectRoots) GlobalSearchScope.filesWithLibrariesScope(project, it) + else GlobalSearchScope.filesWithoutLibrariesScope(project, it) + } + ?: GlobalSearchScope.EMPTY_SCOPE + override fun getSearchScopeByIoFiles(files: Iterable, allowOutOfProjectRoots: Boolean): AbstractProjectFileSearchScope = PsiBasedProjectFileSearchScope( files .mapNotNull { localFileSystem.findFileByPath(it.absolutePath) } - .toList() - .takeIf { it.isNotEmpty() } - ?.let { - if (allowOutOfProjectRoots) GlobalSearchScope.filesWithLibrariesScope(project, it) - else GlobalSearchScope.filesWithoutLibrariesScope(project, it) - } ?: GlobalSearchScope.EMPTY_SCOPE + .toSearchScope(allowOutOfProjectRoots) + ) + + override fun getSearchScopeBySourceFiles(files: Iterable, allowOutOfProjectRoots: Boolean): AbstractProjectFileSearchScope = + PsiBasedProjectFileSearchScope( + files + .mapNotNull { + when (it) { + is KtPsiSourceFile -> it.psiFile.virtualFile + is KtVirtualFileSourceFile -> it.virtualFile + is KtIoFileSourceFile -> localFileSystem.findFileByPath(it.file.absolutePath) + else -> null // TODO: find out whether other use cases should be supported + } + } + .toSearchScope(allowOutOfProjectRoots) ) override fun getSearchScopeByDirectories(directories: Iterable): AbstractProjectFileSearchScope = @@ -108,8 +130,6 @@ open class VfsBasedProjectEnvironment( fileSearchScope: AbstractProjectFileSearchScope ) = FirJavaFacade(firSession, baseModuleData, project.createJavaClassFinder(fileSearchScope.asPsiSearchScope())) - override fun getFileText(filePath: String): String? = - localFileSystem.findFileByPath(filePath)?.inputStream?.reader(Charsets.UTF_8)?.readText() } private fun AbstractProjectFileSearchScope.asPsiSearchScope() = diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipeline.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipeline.kt index 0db0c85eef3..b3ca6807974 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipeline.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipeline.kt @@ -17,6 +17,8 @@ import com.intellij.openapi.vfs.VirtualFileSystem import com.intellij.psi.PsiManager import com.intellij.psi.search.GlobalSearchScope import com.intellij.util.io.URLUtil +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtVirtualFileSourceFile import org.jetbrains.kotlin.analyzer.common.CommonPlatformAnalyzerServices import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension import org.jetbrains.kotlin.backend.jvm.JvmGeneratorExtensionsImpl @@ -105,13 +107,12 @@ fun compileModulesUsingFrontendIrAndLightTree( val moduleConfiguration = compilerConfiguration.copy().applyModuleProperties(module, buildFile).apply { addAll(JVMConfigurationKeys.FRIEND_PATHS, module.getFriendPaths()) } - val platformSources = linkedSetOf() - val commonSources = linkedSetOf() + val platformSources = linkedSetOf() + val commonSources = linkedSetOf() // !! compilerConfiguration.kotlinSourceRoots.forAllFiles(compilerConfiguration, projectEnvironment.project) { virtualFile, isCommon -> - val file = File(virtualFile.canonicalPath ?: virtualFile.path) - if (!file.isFile) error("TODO: better error: file not found $virtualFile") + val file = KtVirtualFileSourceFile(virtualFile) if (isCommon) commonSources.add(file) else platformSources.add(file) } @@ -261,14 +262,14 @@ fun compileModuleToAnalyzedFir( diagnosticsReporter: DiagnosticReporter, performanceManager: CommonCompilerPerformanceManager? ): ModuleCompilerAnalyzedOutput { - var sourcesScope = environment.projectEnvironment.getSearchScopeByIoFiles(input.platformSources) //!! + var sourcesScope = environment.projectEnvironment.getSearchScopeBySourceFiles(input.platformSources) val sessionProvider = FirProjectSessionProvider() val extendedAnalysisMode = input.configuration.getBoolean(CommonConfigurationKeys.USE_FIR_EXTENDED_CHECKERS) val commonSession = runIf( input.commonSources.isNotEmpty() && input.configuration.languageVersionSettings.supportsFeature(LanguageFeature.MultiPlatformProjects) ) { - val commonSourcesScope = environment.projectEnvironment.getSearchScopeByIoFiles(input.commonSources) //!! + val commonSourcesScope = environment.projectEnvironment.getSearchScopeBySourceFiles(input.commonSources) sourcesScope -= commonSourcesScope createSession( "${input.targetId.name}-common", @@ -310,12 +311,11 @@ fun compileModuleToAnalyzedFir( // raw fir val commonRawFir = commonSession?.buildFirViaLightTree( input.commonSources, - environment.projectEnvironment, diagnosticsReporter, countFilesAndLines ) val rawFir = - session.buildFirViaLightTree(input.platformSources, environment.projectEnvironment, diagnosticsReporter, countFilesAndLines) + session.buildFirViaLightTree(input.platformSources, diagnosticsReporter, countFilesAndLines) // resolution commonSession?.apply { diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipelineData.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipelineData.kt index 493556159ad..9262543c686 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipelineData.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipelineData.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.cli.jvm.compiler.pipeline +import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.backend.jvm.JvmGeneratorExtensionsImpl import org.jetbrains.kotlin.codegen.state.GenerationState import org.jetbrains.kotlin.config.CompilerConfiguration @@ -27,9 +28,9 @@ import java.io.File data class ModuleCompilerInput( val targetId: TargetId, val commonPlatform: TargetPlatform, - val commonSources: Collection, + val commonSources: Collection, val platform: TargetPlatform, - val platformSources: Collection, + val platformSources: Collection, val configuration: CompilerConfiguration, val friendFirModules: Collection = emptyList() ) diff --git a/compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/context/CheckerContext.kt b/compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/context/CheckerContext.kt index 264d15a4860..6ebad6f8224 100644 --- a/compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/context/CheckerContext.kt +++ b/compiler/fir/checkers/src/org/jetbrains/kotlin/fir/analysis/checkers/context/CheckerContext.kt @@ -87,7 +87,7 @@ abstract class CheckerContext : MutableDiagnosticContext() { get() = session.languageVersionSettings override val containingFilePath: String? - get() = containingDeclarations.firstOrNull()?.let { (it as? FirFile)?.path } + get() = containingDeclarations.firstOrNull()?.let { (it as? FirFile)?.sourceFile?.path } } /** diff --git a/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/pipeline/buildFir.kt b/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/pipeline/buildFir.kt index 650c18cb078..a015cc3c640 100644 --- a/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/pipeline/buildFir.kt +++ b/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/pipeline/buildFir.kt @@ -5,7 +5,7 @@ package org.jetbrains.kotlin.fir.pipeline -import com.intellij.openapi.util.text.StringUtilRt +import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.diagnostics.DiagnosticReporter import org.jetbrains.kotlin.fir.FirSession import org.jetbrains.kotlin.fir.builder.PsiHandlingMode @@ -14,15 +14,12 @@ import org.jetbrains.kotlin.fir.declarations.FirFile import org.jetbrains.kotlin.fir.lightTree.LightTree2Fir import org.jetbrains.kotlin.fir.resolve.providers.firProvider import org.jetbrains.kotlin.fir.resolve.providers.impl.FirProviderImpl -import org.jetbrains.kotlin.fir.session.environment.AbstractProjectEnvironment import org.jetbrains.kotlin.fir.session.sourcesToPathsMapper import org.jetbrains.kotlin.psi.KtFile -import java.io.File -import java.io.FileNotFoundException +import org.jetbrains.kotlin.readSourceFileWithMapping fun FirSession.buildFirViaLightTree( - files: Collection, - projectEnvironment: AbstractProjectEnvironment, + files: Collection, diagnosticsReporter: DiagnosticReporter? = null, reportFilesAndLines: ((Int, Int) -> Unit)? = null ): List { @@ -32,14 +29,15 @@ fun FirSession.buildFirViaLightTree( val shouldCountLines = (reportFilesAndLines != null) var linesCount = 0 val firFiles = files.map { file -> - val text = projectEnvironment.getFileText(file.absolutePath) ?: throw FileNotFoundException(file.path) - val code = StringUtilRt.convertLineSeparators(text) - if (shouldCountLines) { - linesCount += code.count { it == '\n' } // assuming converted line separators + val (code, linesMapping) = file.getContentsAsStream().reader(Charsets.UTF_8).use { + it.readSourceFileWithMapping() } - builder.buildFirFile(code, file.name, file.path).also { firFile -> + if (shouldCountLines) { + linesCount += linesMapping.lastOffset + } + builder.buildFirFile(code, file, linesMapping).also { firFile -> firProvider.recordFile(firFile) - sourcesToPathsMapper.registerFileSource(firFile.source!!, file.path) + sourcesToPathsMapper.registerFileSource(firFile.source!!, file.path ?: file.name) } } reportFilesAndLines?.invoke(files.count(), linesCount) diff --git a/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/session/environment/AbstractProjectEnvironment.kt b/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/session/environment/AbstractProjectEnvironment.kt index 5cc2be1e743..517e3817041 100644 --- a/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/session/environment/AbstractProjectEnvironment.kt +++ b/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/session/environment/AbstractProjectEnvironment.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.fir.session.environment +import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.fir.FirModuleData import org.jetbrains.kotlin.fir.FirSession import org.jetbrains.kotlin.fir.java.FirJavaFacade @@ -49,6 +50,8 @@ interface AbstractProjectEnvironment { fun getSearchScopeByIoFiles(files: Iterable, allowOutOfProjectRoots: Boolean = false): AbstractProjectFileSearchScope + fun getSearchScopeBySourceFiles(files: Iterable, allowOutOfProjectRoots: Boolean = false): AbstractProjectFileSearchScope + fun getSearchScopeByDirectories(directories: Iterable): AbstractProjectFileSearchScope fun getSearchScopeForProjectLibraries(): AbstractProjectFileSearchScope @@ -60,6 +63,4 @@ interface AbstractProjectEnvironment { baseModuleData: FirModuleData, fileSearchScope: AbstractProjectFileSearchScope ): FirJavaFacade - - fun getFileText(filePath: String): String? } diff --git a/compiler/fir/fir2ir/src/org/jetbrains/kotlin/fir/backend/Fir2IrConverter.kt b/compiler/fir/fir2ir/src/org/jetbrains/kotlin/fir/backend/Fir2IrConverter.kt index 1b4c38b3ffa..735f41496e8 100644 --- a/compiler/fir/fir2ir/src/org/jetbrains/kotlin/fir/backend/Fir2IrConverter.kt +++ b/compiler/fir/fir2ir/src/org/jetbrains/kotlin/fir/backend/Fir2IrConverter.kt @@ -5,6 +5,8 @@ package org.jetbrains.kotlin.fir.backend +import org.jetbrains.kotlin.KtPsiSourceFileLinesMapping +import org.jetbrains.kotlin.KtSourceFileLinesMappingFromLineStartOffsets import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext import org.jetbrains.kotlin.backend.common.ir.BuiltinSymbolsBase @@ -167,7 +169,18 @@ class Fir2IrConverter( private fun registerFileAndClasses(file: FirFile, moduleFragment: IrModuleFragment) { val fileEntry = when (file.origin) { FirDeclarationOrigin.Source -> - file.psi?.let { PsiIrFileEntry(it as KtFile) } ?: NaiveSourceBasedFileEntryImpl(file.path ?: file.name, intArrayOf(0)) + file.psi?.let { PsiIrFileEntry(it as KtFile) } + ?: when (val linesMapping = file.sourceFileLinesMapping) { + is KtSourceFileLinesMappingFromLineStartOffsets -> + NaiveSourceBasedFileEntryImpl( + file.sourceFile?.path ?: file.sourceFile?.name ?: file.name, + linesMapping.lineStartOffsets, + linesMapping.lastOffset + ) + is KtPsiSourceFileLinesMapping -> PsiIrFileEntry(linesMapping.psiFile) + else -> + NaiveSourceBasedFileEntryImpl(file.sourceFile?.path ?: file.sourceFile?.name ?: file.name) + } FirDeclarationOrigin.Synthetic -> NaiveSourceBasedFileEntryImpl(file.name) else -> error("Unsupported file origin: ${file.origin}") } diff --git a/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/LightTree2Fir.kt b/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/LightTree2Fir.kt index 98632052987..66c2795a666 100644 --- a/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/LightTree2Fir.kt +++ b/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/LightTree2Fir.kt @@ -7,9 +7,10 @@ package org.jetbrains.kotlin.fir.lightTree import com.intellij.lang.LighterASTNode import com.intellij.lang.impl.PsiBuilderFactoryImpl -import com.intellij.openapi.util.io.FileUtil -import com.intellij.openapi.vfs.CharsetToolkit import com.intellij.util.diff.FlyweightCapableTreeStructure +import org.jetbrains.kotlin.KtIoFileSourceFile +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtSourceFileLinesMapping import org.jetbrains.kotlin.config.LanguageVersionSettings import org.jetbrains.kotlin.diagnostics.DiagnosticContext import org.jetbrains.kotlin.diagnostics.DiagnosticReporter @@ -22,6 +23,7 @@ import org.jetbrains.kotlin.fir.scopes.FirScopeProvider import org.jetbrains.kotlin.lexer.KotlinLexer import org.jetbrains.kotlin.parsing.KotlinLightParser import org.jetbrains.kotlin.parsing.KotlinParserDefinition +import org.jetbrains.kotlin.readSourceFileWithMapping import java.io.File import java.nio.file.Path @@ -46,7 +48,7 @@ class LightTree2Fir( return builder.lightTree } - fun buildLightTree(code: String): FlyweightCapableTreeStructure { + fun buildLightTree(code: CharSequence): FlyweightCapableTreeStructure { val builder = PsiBuilderFactoryImpl().createBuilder(parserDefinition, makeLexer(), code) KotlinLightParser.parse(builder) return builder.lightTree @@ -58,25 +60,25 @@ class LightTree2Fir( } fun buildFirFile(file: File): FirFile { - val code = FileUtil.loadFile(file, CharsetToolkit.UTF8, true) - return buildFirFile(code, file.name, file.path) + val sourceFile = KtIoFileSourceFile(file) + val (code, linesMapping) = with(file.inputStream().reader(Charsets.UTF_8)) { + this.readSourceFileWithMapping() + } + return buildFirFile(code, sourceFile, linesMapping) } - fun buildFirFile(lightTreeFile: LightTreeFile): FirFile = with(lightTreeFile) { + fun buildFirFile( + lightTree: FlyweightCapableTreeStructure, + sourceFile: KtSourceFile, + linesMapping: KtSourceFileLinesMapping + ): FirFile = DeclarationsConverter( session, scopeProvider, lightTree, diagnosticsReporter = diagnosticsReporter, - diagnosticContext = makeDiagnosticContext(path) - ).convertFile(lightTree.root, fileName, path) - } + diagnosticContext = makeDiagnosticContext(sourceFile.path) + ).convertFile(lightTree.root, sourceFile, linesMapping) - fun buildFirFile(code: String, fileName: String, path: String?): FirFile { - val lightTree = buildLightTree(code) - - return DeclarationsConverter( - session, scopeProvider, lightTree, diagnosticsReporter = diagnosticsReporter, - diagnosticContext = makeDiagnosticContext(path) - ).convertFile(lightTree.root, fileName, path) - } + fun buildFirFile(code: CharSequence, sourceFile: KtSourceFile, linesMapping: KtSourceFileLinesMapping): FirFile = + buildFirFile(buildLightTree(code), sourceFile, linesMapping) private fun makeDiagnosticContext(path: String?) = if (diagnosticsReporter == null) null else object : DiagnosticContext { @@ -86,9 +88,3 @@ class LightTree2Fir( } } -data class LightTreeFile( - val lightTree: FlyweightCapableTreeStructure, - val fileName: String, - val path: String? -) - diff --git a/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/converter/DeclarationsConverter.kt b/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/converter/DeclarationsConverter.kt index 83b72c1998d..cd85b3c6c0f 100644 --- a/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/converter/DeclarationsConverter.kt +++ b/compiler/fir/raw-fir/light-tree2fir/src/org/jetbrains/kotlin/fir/lightTree/converter/DeclarationsConverter.kt @@ -88,7 +88,7 @@ class DeclarationsConverter( * [org.jetbrains.kotlin.parsing.KotlinParsing.parseFile] * [org.jetbrains.kotlin.parsing.KotlinParsing.parsePreamble] */ - fun convertFile(file: LighterASTNode, fileName: String = "", filePath: String?): FirFile { + fun convertFile(file: LighterASTNode, sourceFile: KtSourceFile, linesMapping: KtSourceFileLinesMapping): FirFile { if (file.tokenType != KT_FILE) { //TODO throw error throw Exception() @@ -123,8 +123,9 @@ class DeclarationsConverter( source = file.toFirSourceElement() origin = FirDeclarationOrigin.Source moduleData = baseModuleData - name = fileName - path = filePath + name = sourceFile.name + this.sourceFile = sourceFile + this.sourceFileLinesMapping = linesMapping this.packageDirective = packageDirective ?: buildPackageDirective { packageFqName = context.packageFqName } annotations += fileAnnotationList imports += importList diff --git a/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/TotalKotlinTest.kt b/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/TotalKotlinTest.kt index dd8fb5f625e..97ea184e309 100644 --- a/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/TotalKotlinTest.kt +++ b/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/TotalKotlinTest.kt @@ -10,11 +10,15 @@ import com.intellij.openapi.vfs.CharsetToolkit import com.intellij.psi.impl.DebugUtil import com.intellij.testFramework.TestDataPath import com.intellij.util.PathUtil +import org.jetbrains.kotlin.KtIoFileSourceFile +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtSourceFileLinesMapping import org.jetbrains.kotlin.fir.FirRenderer import org.jetbrains.kotlin.fir.builder.AbstractRawFirBuilderTestCase import org.jetbrains.kotlin.fir.builder.StubFirScopeProvider import org.jetbrains.kotlin.fir.session.FirSessionFactory import org.jetbrains.kotlin.psi.KtFile +import org.jetbrains.kotlin.readSourceFileWithMapping import org.jetbrains.kotlin.test.JUnit3RunnerWithInners import org.junit.runner.RunWith import java.io.File @@ -33,12 +37,15 @@ class TotalKotlinTest : AbstractRawFirBuilderTestCase() { } } - private fun generateFirFromLightTree(onlyLightTree: Boolean, converter: LightTree2Fir, text: String, fileName: String, filePath: String) { + private fun generateFirFromLightTree( + onlyLightTree: Boolean, converter: LightTree2Fir, + text: CharSequence, sourceFile: KtSourceFile, linesMapping: KtSourceFileLinesMapping + ) { if (onlyLightTree) { val lightTree = LightTree2Fir.buildLightTree(text) DebugUtil.lightTreeToString(lightTree, false) } else { - val firFile = converter.buildFirFile(text, fileName, filePath) + val firFile = converter.buildFirFile(text, sourceFile, linesMapping) StringBuilder().also { FirRenderer(it).visitFile(firFile) }.toString() } } @@ -57,9 +64,12 @@ class TotalKotlinTest : AbstractRawFirBuilderTestCase() { if (onlyLightTree) println("LightTree generation") else println("Fir from LightTree converter") println("BASE PATH: $path") path.walkTopDown { - val text = FileUtil.loadFile(it, CharsetToolkit.UTF8, true).trim() + val sourceFile = KtIoFileSourceFile(it) + val (code, linesMapping) = with(it.inputStream().reader(Charsets.UTF_8)) { + this.readSourceFileWithMapping() + } time += measureNanoTime { - generateFirFromLightTree(onlyLightTree, lightTreeConverter, text, it.name, it.path) + generateFirFromLightTree(onlyLightTree, lightTreeConverter, code, sourceFile, linesMapping) } counter++ diff --git a/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/compare/TreesCompareTest.kt b/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/compare/TreesCompareTest.kt index 36326246e9c..d80d2b86c15 100644 --- a/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/compare/TreesCompareTest.kt +++ b/compiler/fir/raw-fir/light-tree2fir/tests/org/jetbrains/kotlin/fir/lightTree/compare/TreesCompareTest.kt @@ -10,6 +10,8 @@ import com.intellij.openapi.vfs.CharsetToolkit import com.intellij.testFramework.TestDataPath import com.intellij.util.PathUtil import junit.framework.TestCase +import org.jetbrains.kotlin.KtInMemoryTextSourceFile +import org.jetbrains.kotlin.KtIoFileSourceFile import org.jetbrains.kotlin.checkers.BaseDiagnosticsTest.Companion.DIAGNOSTIC_IN_TESTDATA_PATTERN import org.jetbrains.kotlin.fir.FirRenderer import org.jetbrains.kotlin.fir.builder.AbstractRawFirBuilderTestCase @@ -19,7 +21,9 @@ import org.jetbrains.kotlin.fir.lightTree.walkTopDown import org.jetbrains.kotlin.fir.lightTree.walkTopDownWithTestData import org.jetbrains.kotlin.fir.session.FirSessionFactory import org.jetbrains.kotlin.psi.* +import org.jetbrains.kotlin.readSourceFileWithMapping import org.jetbrains.kotlin.test.JUnit3RunnerWithInners +import org.jetbrains.kotlin.toSourceLinesMapping import org.junit.runner.RunWith import java.io.File @@ -61,15 +65,17 @@ class TreesCompareTest : AbstractRawFirBuilderTestCase() { diagnosticsReporter = null ) compareBase(System.getProperty("user.dir"), withTestData = false) { file -> - val text = FileUtil.loadFile(file, CharsetToolkit.UTF8, true).trim() + val (text, linesMapping) = with(file.inputStream().reader(Charsets.UTF_8)) { + this.readSourceFileWithMapping() + } //psi - val ktFile = createPsiFile(FileUtil.getNameWithoutExtension(PathUtil.getFileName(file.path)), text) as KtFile + val ktFile = createPsiFile(FileUtil.getNameWithoutExtension(PathUtil.getFileName(file.path)), text.toString().trim()) as KtFile val firFileFromPsi = ktFile.toFirFile() val treeFromPsi = StringBuilder().also { FirRenderer(it).visitFile(firFileFromPsi) }.toString() //light tree - val firFileFromLightTree = lightTreeConverter.buildFirFile(text, file.name, file.path) + val firFileFromLightTree = lightTreeConverter.buildFirFile(text, KtIoFileSourceFile(file), linesMapping) val treeFromLightTree = StringBuilder().also { FirRenderer(it).visitFile(firFileFromLightTree) }.toString() return@compareBase treeFromLightTree == treeFromPsi @@ -100,7 +106,12 @@ class TreesCompareTest : AbstractRawFirBuilderTestCase() { .replace("".toRegex(), "") //light tree - val firFileFromLightTree = lightTreeConverter.buildFirFile(text, file.name, file.path) + val firFileFromLightTree = + lightTreeConverter.buildFirFile( + text, + KtInMemoryTextSourceFile(file.name, file.path, text), + text.toSourceLinesMapping() + ) val treeFromLightTree = StringBuilder().also { FirRenderer(it).visitFile(firFileFromLightTree) }.toString() .replace("".toRegex(), "") diff --git a/compiler/fir/raw-fir/psi2fir/src/org/jetbrains/kotlin/fir/builder/RawFirBuilder.kt b/compiler/fir/raw-fir/psi2fir/src/org/jetbrains/kotlin/fir/builder/RawFirBuilder.kt index 93090e20674..e72441e527e 100644 --- a/compiler/fir/raw-fir/psi2fir/src/org/jetbrains/kotlin/fir/builder/RawFirBuilder.kt +++ b/compiler/fir/raw-fir/psi2fir/src/org/jetbrains/kotlin/fir/builder/RawFirBuilder.kt @@ -911,7 +911,8 @@ open class RawFirBuilder( moduleData = baseModuleData origin = FirDeclarationOrigin.Source name = file.name - path = file.virtualFile?.path + sourceFile = KtPsiSourceFile(file) + sourceFileLinesMapping = KtPsiSourceFileLinesMapping(file) packageDirective = buildPackageDirective { packageFqName = context.packageFqName source = file.packageDirective?.toKtPsiSourceElement() diff --git a/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/FirFile.kt b/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/FirFile.kt index 55c66a3fe3e..3c31c9861ff 100644 --- a/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/FirFile.kt +++ b/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/FirFile.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.fir.declarations import org.jetbrains.kotlin.KtSourceElement +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtSourceFileLinesMapping import org.jetbrains.kotlin.fir.FirElement import org.jetbrains.kotlin.fir.FirModuleData import org.jetbrains.kotlin.fir.FirPackageDirective @@ -29,7 +31,8 @@ abstract class FirFile : FirDeclaration() { abstract val imports: List abstract val declarations: List abstract val name: String - abstract val path: String? + abstract val sourceFile: KtSourceFile? + abstract val sourceFileLinesMapping: KtSourceFileLinesMapping? abstract override val symbol: FirFileSymbol override fun accept(visitor: FirVisitor, data: D): R = visitor.visitFile(this, data) diff --git a/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/builder/FirFileBuilder.kt b/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/builder/FirFileBuilder.kt index 05ebe41fb65..27da97e5438 100644 --- a/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/builder/FirFileBuilder.kt +++ b/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/builder/FirFileBuilder.kt @@ -7,6 +7,8 @@ package org.jetbrains.kotlin.fir.declarations.builder import kotlin.contracts.* import org.jetbrains.kotlin.KtSourceElement +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtSourceFileLinesMapping import org.jetbrains.kotlin.fir.FirModuleData import org.jetbrains.kotlin.fir.FirPackageDirective import org.jetbrains.kotlin.fir.builder.FirAnnotationContainerBuilder @@ -39,7 +41,8 @@ class FirFileBuilder : FirAnnotationContainerBuilder { val imports: MutableList = mutableListOf() val declarations: MutableList = mutableListOf() lateinit var name: String - var path: String? = null + var sourceFile: KtSourceFile? = null + var sourceFileLinesMapping: KtSourceFileLinesMapping? = null override fun build(): FirFile { return FirFileImpl( @@ -53,7 +56,8 @@ class FirFileBuilder : FirAnnotationContainerBuilder { imports, declarations, name, - path, + sourceFile, + sourceFileLinesMapping, ) } diff --git a/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/impl/FirFileImpl.kt b/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/impl/FirFileImpl.kt index a638f5daedb..059ebaee09e 100644 --- a/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/impl/FirFileImpl.kt +++ b/compiler/fir/tree/gen/org/jetbrains/kotlin/fir/declarations/impl/FirFileImpl.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.fir.declarations.impl import org.jetbrains.kotlin.KtSourceElement +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtSourceFileLinesMapping import org.jetbrains.kotlin.fir.FirModuleData import org.jetbrains.kotlin.fir.FirPackageDirective import org.jetbrains.kotlin.fir.declarations.FirDeclaration @@ -35,7 +37,8 @@ internal class FirFileImpl( override val imports: MutableList, override val declarations: MutableList, override val name: String, - override val path: String?, + override val sourceFile: KtSourceFile?, + override val sourceFileLinesMapping: KtSourceFileLinesMapping?, ) : FirFile() { override val symbol: FirFileSymbol = FirFileSymbol() diff --git a/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/NodeConfigurator.kt b/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/NodeConfigurator.kt index 17d726e7f3b..fe991e7a295 100644 --- a/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/NodeConfigurator.kt +++ b/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/NodeConfigurator.kt @@ -415,7 +415,8 @@ object NodeConfigurator : AbstractFieldConfigurator(FirTreeBuild +fieldList(import).withTransform() +declarations.withTransform() +stringField("name") - +stringField("path", nullable = true) + +field("sourceFile", sourceFileType, nullable = true) + +field("sourceFileLinesMapping", sourceFileLinesMappingType, nullable = true) +symbol("FirFileSymbol") } diff --git a/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/Types.kt b/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/Types.kt index 97b69d4093a..f3c78760229 100644 --- a/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/Types.kt +++ b/compiler/fir/tree/tree-generator/src/org/jetbrains/kotlin/fir/tree/generator/Types.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.fir.tree.generator import org.jetbrains.kotlin.KtSourceElement +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtSourceFileLinesMapping import org.jetbrains.kotlin.contracts.description.EventOccurrencesRange import org.jetbrains.kotlin.descriptors.ClassKind import org.jetbrains.kotlin.descriptors.Modality @@ -23,6 +25,8 @@ import org.jetbrains.kotlin.types.SmartcastStability import org.jetbrains.kotlin.types.Variance val sourceElementType = type(KtSourceElement::class) +val sourceFileType = type(KtSourceFile::class) +val sourceFileLinesMappingType = type(KtSourceFileLinesMapping::class) val jumpTargetType = type("fir", "FirTarget") val constKindType = type("types", "ConstantValueKind") val operationType = type("fir.expressions", "FirOperation") diff --git a/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceElement.kt b/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceElement.kt index 9178d1a7034..b3580609528 100644 --- a/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceElement.kt +++ b/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceElement.kt @@ -14,9 +14,6 @@ import com.intellij.psi.PsiFile import com.intellij.psi.PsiWhiteSpace import com.intellij.psi.tree.IElementType import com.intellij.util.diff.FlyweightCapableTreeStructure -import java.io.Closeable -import java.io.File -import java.io.InputStreamReader sealed class KtSourceElementKind @@ -444,90 +441,3 @@ inline fun LighterASTNode.toKtLightSourceElement( endOffset: Int = this.endOffset ): KtLightSourceElement = KtLightSourceElement(this, startOffset, endOffset, tree, kind) - -class KtSourceFilePosition(val line: Int, val column: Int, val lineContent: String?) { - - // NOTE: This method is used for presenting positions to the user - override fun toString(): String = if (line < 0) "(offset: $column line unknown)" else "($line,$column)" - - companion object { - val NONE = KtSourceFilePosition(-1, -1, null) - } -} - -class SequentialFilePositionFinder(file: File) : Closeable { - - private var reader: InputStreamReader = file.reader(/* TODO: select proper charset */) - - private var currentLineContent: String? = null - private val buffer = CharArray(255) - private var bufLength = -1 - private var bufPos = 0 - private var endOfStream = false - private var skipNextLf = false - - private var charsRead = 0 - private var currentLine = 0 - - // assuming that if called multiple times, calls should be sorted by ascending offset - fun findNextPosition(offset: Int, withLineContents: Boolean = true): KtSourceFilePosition { - assert(offset >= charsRead - (currentLineContent?.length ?: 0)) - - fun posInCurrentLine(): KtSourceFilePosition? { - val col = offset - (charsRead - currentLineContent!!.length - 1)/* beginning of line offset */ + 1 /* col is 1-based */ - return if (col <= currentLineContent!!.length) - KtSourceFilePosition(currentLine, col, if (withLineContents) currentLineContent else null) - else null - } - - if (offset < charsRead) { - return posInCurrentLine()!! - } - - while (true) { - if (currentLineContent == null) { - currentLineContent = readNextLine() - } - - posInCurrentLine()?.let { return@findNextPosition it } - - if (endOfStream) return KtSourceFilePosition(-1, offset, if (withLineContents) currentLineContent else null) - - currentLineContent = null - } - } - - private fun readNextLine() = buildString { - while (true) { - if (bufPos >= bufLength) { - bufLength = reader.read(buffer) - bufPos = 0 - if (bufLength < 0) { - endOfStream = true - break - } - } else { - val c = buffer[bufPos++] - charsRead++ - when { - c == '\n' && skipNextLf -> { - skipNextLf = false - } - c == '\n' || c == '\r' -> { - currentLine++ - skipNextLf = c == '\r' - break - } - else -> { - append(c) - skipNextLf = false - } - } - } - } - } - - override fun close() { - reader.close() - } -} diff --git a/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceFile.kt b/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceFile.kt new file mode 100644 index 00000000000..d481e64c594 --- /dev/null +++ b/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceFile.kt @@ -0,0 +1,57 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin + +import com.intellij.openapi.util.io.FileUtilRt +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.psi.PsiFile +import java.io.ByteArrayInputStream +import java.io.File +import java.io.InputStream + +interface KtSourceFile { + val name: String + val path: String? + + fun getContentsAsStream(): InputStream +} + +class KtPsiSourceFile(val psiFile: PsiFile) : KtSourceFile { + override val name: String + get() = psiFile.name + + override val path: String? + get() = psiFile.virtualFile?.path + + override fun getContentsAsStream(): InputStream = psiFile.virtualFile.inputStream +} + +class KtVirtualFileSourceFile(val virtualFile: VirtualFile) : KtSourceFile { + override val name: String + get() = virtualFile.name + + override val path: String + get() = virtualFile.path + + override fun getContentsAsStream(): InputStream = virtualFile.inputStream +} + +class KtIoFileSourceFile(val file: File) : KtSourceFile { + override val name: String + get() = file.name + override val path: String + get() = FileUtilRt.toSystemIndependentName(file.path) + + override fun getContentsAsStream(): InputStream = file.inputStream() +} + +class KtInMemoryTextSourceFile( + override val name: String, + override val path: String?, + val text: CharSequence +) : KtSourceFile { + override fun getContentsAsStream(): InputStream = ByteArrayInputStream(text.toString().toByteArray()) +} diff --git a/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceFileLinesMapping.kt b/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceFileLinesMapping.kt new file mode 100644 index 00000000000..5630d9c08a5 --- /dev/null +++ b/compiler/frontend.common/src/org/jetbrains/kotlin/KtSourceFileLinesMapping.kt @@ -0,0 +1,126 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin + +import com.intellij.openapi.editor.Document +import com.intellij.psi.PsiFile +import java.io.InputStreamReader + +interface KtSourceFileLinesMapping { + fun getLineStartOffset(line: Int): Int + fun getLineAndColumnByOffset(offset: Int): Pair + fun getLineByOffset(offset: Int): Int + + val lastOffset: Int + val linesCount: Int +} + +class KtPsiSourceFileLinesMapping(val psiFile: PsiFile) : KtSourceFileLinesMapping { + private val document: Document? by lazy { psiFile.viewProvider.document } + + override fun getLineStartOffset(line: Int): Int = + document?.getLineStartOffset(line) ?: -1 + + override fun getLineAndColumnByOffset(offset: Int): Pair = + document?.let { + val lineNumber = it.getLineNumber(offset) + val lineStartOffset = it.getLineStartOffset(lineNumber) + lineNumber to offset - lineStartOffset + } ?: (-1 to -1) + + override fun getLineByOffset(offset: Int): Int = + document?.getLineNumber(offset) ?: -1 + + override val lastOffset: Int + get() = document?.textLength ?: -1 + + override val linesCount: Int + get() = document?.lineCount ?: 0 +} + +open class KtSourceFileLinesMappingFromLineStartOffsets( + val lineStartOffsets: IntArray, override val lastOffset: Int +) : KtSourceFileLinesMapping { + override fun getLineStartOffset(line: Int): Int = lineStartOffsets[line] + + override fun getLineAndColumnByOffset(offset: Int): Pair { + val lineNumber = getLineByOffset(offset) + if (lineNumber < 0) return -1 to -1 + val lineStartOffset = lineStartOffsets[lineNumber] + return lineNumber to offset - lineStartOffset + } + + override fun getLineByOffset(offset: Int): Int { + val index = lineStartOffsets.binarySearch(offset) + return if (index >= 0) index else -index - 2 + } + + override val linesCount: Int + get() = lineStartOffsets.size +} + +/** + * Reads file contents from reader, converts line separators and calculates source lines to file offsets mapping + * + * Returns KtSourceFileLinesMapping and char sequence (StringBuilder to avoid premature copying) containing converted text + * The separators are converted similarly to the com.intellij.openapi.util.text.StringUtilRt algorithms + */ +fun InputStreamReader.readSourceFileWithMapping(): Pair { + val buffer = CharArray(255) + var bufLength = -1 + var bufPos = 0 + var skipNextLf = false + + var charsRead = 0 + + val lineOffsets = mutableListOf(0) // TODO: consider using implicit first line offset (needs to be handled properly in IR) + val sb = StringBuilder() + + while (true) { + if (bufPos >= bufLength) { + bufLength = read(buffer) + bufPos = 0 + if (bufLength < 0) { + break + } + } else { + val c = buffer[bufPos++] + charsRead++ + when { + c == '\n' && skipNextLf -> { + lineOffsets[lineOffsets.size - 1] = charsRead + skipNextLf = false + } + c == '\n' || c == '\r' -> { + sb.append('\n') + lineOffsets.add(charsRead) + skipNextLf = c == '\r' + } + else -> { + sb.append(c) + skipNextLf = false + } + } + } + } + + return sb to KtSourceFileLinesMappingFromLineStartOffsets(lineOffsets.toIntArray(), charsRead) +} + +/** + * Extracts source lines to offsets mapping from text + * + * intended for using mainly in tests, so no care is taken about performance or possible corner cases + */ +fun CharSequence.toSourceLinesMapping(): KtSourceFileLinesMapping { + val lineOffsets = mutableListOf(0) + var offset = 0 + for (c in this) { + offset++ + if (c == '\n') lineOffsets.add(offset) + } + return KtSourceFileLinesMappingFromLineStartOffsets(lineOffsets.toIntArray(), offset) +} diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt index 71c8c3174ea..ca46740b2bb 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalFirJvmCompilerRunner.kt @@ -8,6 +8,9 @@ package org.jetbrains.kotlin.incremental import com.intellij.ide.highlighter.JavaFileType import com.intellij.openapi.util.Disposer import com.intellij.psi.PsiJavaModule +import org.jetbrains.kotlin.KtIoFileSourceFile +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtVirtualFileSourceFile import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension import org.jetbrains.kotlin.backend.jvm.JvmGeneratorExtensionsImpl import org.jetbrains.kotlin.backend.jvm.serialization.JvmIdSignatureDescriptor @@ -99,7 +102,7 @@ class IncrementalFirJvmCompilerRunner( val moduleName = args.moduleName ?: JvmProtoBufUtil.DEFAULT_MODULE_NAME val targetId = TargetId(moduleName, "java-production") // TODO: get rid of magic constant - val dirtySources = linkedSetOf().apply { addAll(sourcesToCompile) } + val dirtySources = linkedSetOf().apply { sourcesToCompile.forEach { add(KtIoFileSourceFile(it)) } } // TODO: probably shoudl be passed along with sourcesToCompile // TODO: file path normalization @@ -175,12 +178,11 @@ class IncrementalFirJvmCompilerRunner( createProjectEnvironment(configuration, rootDisposable, EnvironmentConfigFiles.JVM_CONFIG_FILES, messageCollector) // -sources - val allPlatformSourceFiles = linkedSetOf() // TODO: get from caller - val allCommonSourceFiles = linkedSetOf() + val allPlatformSourceFiles = linkedSetOf() // TODO: get from caller + val allCommonSourceFiles = linkedSetOf() configuration.kotlinSourceRoots.forAllFiles(configuration, projectEnvironment.project) { virtualFile, isCommon -> - val file = File(virtualFile.canonicalPath ?: virtualFile.path) - if (!file.isFile) error("TODO: better error: file not found $virtualFile") + val file = KtVirtualFileSourceFile(virtualFile) if (isCommon) allCommonSourceFiles.add(file) else allPlatformSourceFiles.add(file) } @@ -203,8 +205,8 @@ class IncrementalFirJvmCompilerRunner( val compilerInput = ModuleCompilerInput( targetId, - CommonPlatforms.defaultCommonPlatform, dirtySources.filter { it in allCommonSourceFiles }, - JvmPlatforms.unspecifiedJvmPlatform, dirtySources.filter { it in allPlatformSourceFiles }, + CommonPlatforms.defaultCommonPlatform, allCommonSourceFiles.filter { dirtySources.any { df -> df.path == it.path } }, + JvmPlatforms.unspecifiedJvmPlatform, allPlatformSourceFiles.filter { dirtySources.any { df -> df.path == it.path } }, configuration ) @@ -227,7 +229,10 @@ class IncrementalFirJvmCompilerRunner( mainClassFqName = findMainClass(analysisResults.fir) } - allCompiledSources.addAll(dirtySources) + // TODO: switch the whole IC to KtSourceFile instead of FIle + dirtySources.forEach { + allCompiledSources.add(File(it.path!!)) + } if (diagnosticsReporter.hasErrors) { diagnosticsReporter.reportToMessageCollector(messageCollector, renderDiagnosticName) @@ -250,7 +255,9 @@ class IncrementalFirJvmCompilerRunner( } } caches.inputsCache.removeOutputForSourceFiles(newDirtySources) - dirtySources.addAll(newDirtySources) + newDirtySources.forEach { + dirtySources.add(KtIoFileSourceFile(it)) + } projectEnvironment.localFileSystem.refresh(false) } } diff --git a/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/Utils.kt b/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/Utils.kt index 902146b4cde..2e1c6a8bba8 100644 --- a/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/Utils.kt +++ b/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/Utils.kt @@ -18,14 +18,9 @@ package org.jetbrains.kotlin.backend.common import org.jetbrains.kotlin.AbstractKtSourceElement import org.jetbrains.kotlin.KtOffsetsOnlySourceElement -import org.jetbrains.kotlin.KtRealPsiSourceElement -import org.jetbrains.kotlin.KtSourceElement -import org.jetbrains.kotlin.backend.common.psi.PsiSourceManager import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET -import org.jetbrains.kotlin.ir.declarations.IrDeclaration import org.jetbrains.kotlin.ir.declarations.IrFile -import org.jetbrains.kotlin.ir.util.render fun CommonBackendContext.reportWarning(message: String, irFile: IrFile?, irElement: IrElement) { report(irElement, irFile, message, false) @@ -37,12 +32,6 @@ fun MutableList.pop() = this.removeAt(size - 1) fun MutableList.peek(): E? = if (size == 0) null else this[size - 1] -fun findKtSourceElement(irElement: IrElement, irDeclaration: IrDeclaration): KtSourceElement { - val psiElement = PsiSourceManager.findPsiElement(irElement, irDeclaration) - ?: throw AssertionError("No PsiElement found for '${irElement.render()}'") - return KtRealPsiSourceElement(psiElement) -} - fun IrElement.sourceElement(): AbstractKtSourceElement? = if (startOffset != UNDEFINED_OFFSET) KtOffsetsOnlySourceElement(this.startOffset, this.endOffset) else null diff --git a/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/ir/JvmIrUtils.kt b/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/ir/JvmIrUtils.kt index 66118e6572a..93caf75b6ff 100644 --- a/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/ir/JvmIrUtils.kt +++ b/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/ir/JvmIrUtils.kt @@ -285,7 +285,7 @@ fun IrFile.getKtFile(): KtFile? = fun IrFile.getIoFile(): File? = when (val fe = fileEntry) { is PsiIrFileEntry -> fe.psiFile.virtualFile?.path?.let(::File) - else -> File(fe.name).takeIf { it.exists() } + else -> File(fe.name) } inline fun IrElement.hasChild(crossinline block: (IrElement) -> Boolean): Boolean { diff --git a/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/util/AdditionalIrUtils.kt b/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/util/AdditionalIrUtils.kt index 9a16228659a..69edeaf473b 100644 --- a/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/util/AdditionalIrUtils.kt +++ b/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/util/AdditionalIrUtils.kt @@ -186,7 +186,8 @@ val IrFileEntry.lineStartOffsets: IntArray class NaiveSourceBasedFileEntryImpl( override val name: String, - private val lineStartOffsets: IntArray = intArrayOf() + private val lineStartOffsets: IntArray = intArrayOf(), + override val maxOffset: Int = UNDEFINED_OFFSET ) : IrFileEntry { val lineStartOffsetsAreEmpty: Boolean get() = lineStartOffsets.isEmpty() @@ -215,15 +216,19 @@ class NaiveSourceBasedFileEntryImpl( if (offset == SYNTHETIC_OFFSET) return 0 if (offset < 0) return -1 val lineNumber = getLineNumber(offset) - return offset - lineStartOffsets[lineNumber] + return if (lineNumber < 0) -1 else offset - lineStartOffsets[lineNumber] } - override val maxOffset: Int - get() = UNDEFINED_OFFSET - - override fun getSourceRangeInfo(beginOffset: Int, endOffset: Int): SourceRangeInfo { - return SourceRangeInfo(name, beginOffset, -1, -1, endOffset, -1, -1) - } + override fun getSourceRangeInfo(beginOffset: Int, endOffset: Int): SourceRangeInfo = + SourceRangeInfo( + filePath = name, + startOffset = beginOffset, + startLineNumber = getLineNumber(beginOffset), + startColumnNumber = getColumnNumber(beginOffset), + endOffset = endOffset, + endLineNumber = getLineNumber(endOffset), + endColumnNumber = getColumnNumber(endOffset) + ) } private fun IrClass.getPropertyDeclaration(name: String): IrProperty? { diff --git a/compiler/test-infrastructure-utils/tests/org/jetbrains/kotlin/sourceFiles/LightTreeFile.kt b/compiler/test-infrastructure-utils/tests/org/jetbrains/kotlin/sourceFiles/LightTreeFile.kt new file mode 100644 index 00000000000..c0b4656ccbf --- /dev/null +++ b/compiler/test-infrastructure-utils/tests/org/jetbrains/kotlin/sourceFiles/LightTreeFile.kt @@ -0,0 +1,17 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.sourceFiles + +import com.intellij.lang.LighterASTNode +import com.intellij.util.diff.FlyweightCapableTreeStructure +import org.jetbrains.kotlin.KtSourceFile +import org.jetbrains.kotlin.KtSourceFileLinesMapping + +data class LightTreeFile( + val lightTree: FlyweightCapableTreeStructure, + val sourceFile: KtSourceFile, + val linesMapping: KtSourceFileLinesMapping +) diff --git a/compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/SourceFileProvider.kt b/compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/SourceFileProvider.kt index edfeadab85b..41c1f932814 100644 --- a/compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/SourceFileProvider.kt +++ b/compiler/test-infrastructure/tests/org/jetbrains/kotlin/test/services/SourceFileProvider.kt @@ -6,11 +6,13 @@ package org.jetbrains.kotlin.test.services import com.intellij.openapi.project.Project +import org.jetbrains.kotlin.KtInMemoryTextSourceFile import org.jetbrains.kotlin.fir.lightTree.LightTree2Fir -import org.jetbrains.kotlin.fir.lightTree.LightTreeFile import org.jetbrains.kotlin.psi.KtFile +import org.jetbrains.kotlin.sourceFiles.LightTreeFile import org.jetbrains.kotlin.test.model.TestFile import org.jetbrains.kotlin.test.util.KtTestUtil +import org.jetbrains.kotlin.toSourceLinesMapping import java.io.File abstract class SourceFilePreprocessor(val testServices: TestServices) { @@ -101,9 +103,10 @@ fun SourceFileProvider.getKtFilesForSourceFiles(testFiles: Collection, fun SourceFileProvider.getLightTreeKtFileForSourceFile(testFile: TestFile): LightTreeFile { val shortName = testFile.name.substringAfterLast('/').substringAfterLast('\\') - val file = getRealFileForSourceFile(testFile) - val lightTree = LightTree2Fir.buildLightTree(file.readText()) - return LightTreeFile(lightTree, shortName, "/$shortName") // emulating behavior of KtTestUtil.createFile so path looks the same in testdata + val sourceFile = KtInMemoryTextSourceFile(shortName, "/$shortName", getContentOfSourceFile(testFile)) + val linesMapping = sourceFile.text.toSourceLinesMapping() + val lightTree = LightTree2Fir.buildLightTree(sourceFile.text) + return LightTreeFile(lightTree, sourceFile, linesMapping) } fun SourceFileProvider.getLightTreeFilesForSourceFiles(testFiles: Collection): Map { diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/classic/ClassicJvmBackendFacade.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/classic/ClassicJvmBackendFacade.kt index aa650d468d6..9748a41c6c0 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/classic/ClassicJvmBackendFacade.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/classic/ClassicJvmBackendFacade.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.test.backend.classic +import org.jetbrains.kotlin.KtPsiSourceFile import org.jetbrains.kotlin.codegen.ClassBuilderFactories import org.jetbrains.kotlin.codegen.DefaultCodegenFactory import org.jetbrains.kotlin.codegen.KotlinCodegenFacade @@ -46,7 +47,7 @@ class ClassicJvmBackendFacade( javaCompilerFacade.compileJavaFiles(module, configuration, generationState.factory) return BinaryArtifacts.Jvm( generationState.factory, - psiFiles.map { SourceFileInfo(it, JvmFileClassUtil.getFileClassInfoNoResolve(it)) } + psiFiles.map { SourceFileInfo(KtPsiSourceFile(it), JvmFileClassUtil.getFileClassInfoNoResolve(it)) } ) } } diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/IrTextDumpHandler.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/IrTextDumpHandler.kt index 1d822b59854..70256c3a85d 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/IrTextDumpHandler.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/IrTextDumpHandler.kt @@ -14,7 +14,10 @@ import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerDesc import org.jetbrains.kotlin.ir.declarations.IrClass import org.jetbrains.kotlin.ir.declarations.IrFile import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl -import org.jetbrains.kotlin.ir.util.* +import org.jetbrains.kotlin.ir.util.DeclarationStubGenerator +import org.jetbrains.kotlin.ir.util.SymbolTable +import org.jetbrains.kotlin.ir.util.dump +import org.jetbrains.kotlin.ir.util.dumpTreesFromLineNumber import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.psi2ir.generators.DeclarationStubGeneratorImpl @@ -46,7 +49,7 @@ class IrTextDumpHandler(testServices: TestServices) : AbstractIrHandler(testServ } fun List.groupWithTestFiles(module: TestModule): List> = mapNotNull { irFile -> - val name = irFile.fileEntry.name.removePrefix("/") + val name = File(irFile.fileEntry.name).name val testFile = module.files.firstOrNull { it.name == name } testFile to irFile } diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/JvmBoxRunner.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/JvmBoxRunner.kt index 66c266dc983..1682f819f13 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/JvmBoxRunner.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/JvmBoxRunner.kt @@ -61,7 +61,7 @@ open class JvmBoxRunner(testServices: TestServices) : JvmBinaryArtifactHandler(t val classLoader = createAndVerifyClassLoader(module, info.classFileFactory, reportProblems) try { for (fileInfo in fileInfos) { - if (fileContainsBoxMethod(fileInfo.file)) { + if (fileContainsBoxMethod(fileInfo.sourceFile)) { boxMethodFound = true callBoxMethodAndCheckResultWithCleanup( fileInfo.info, diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/SMAPDumpHandler.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/SMAPDumpHandler.kt index b330f91de84..7072d1869e1 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/SMAPDumpHandler.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/SMAPDumpHandler.kt @@ -19,6 +19,7 @@ import org.jetbrains.kotlin.test.services.TestServices import org.jetbrains.kotlin.test.services.moduleStructure import org.jetbrains.kotlin.test.utils.MultiModuleInfoDumper import org.jetbrains.kotlin.test.utils.withExtension +import java.io.File class SMAPDumpHandler(testServices: TestServices) : JvmBinaryArtifactHandler(testServices) { companion object { @@ -39,7 +40,7 @@ class SMAPDumpHandler(testServices: TestServices) : JvmBinaryArtifactHandler(tes val originalFileNames = module.files.map { it.name } val compiledSmaps = CommonSMAPTestUtil.extractSMAPFromClasses(info.classFileFactory.getClassFiles()).mapNotNull { - val name = it.sourceFile.removePrefix("/") + val name = File(it.sourceFile).name val index = originalFileNames.indexOf(name) val testFile = module.files[index] if (NO_SMAP_DUMP in testFile.directives) return@mapNotNull null @@ -57,7 +58,7 @@ class SMAPDumpHandler(testServices: TestServices) : JvmBinaryArtifactHandler(tes dumper.builderForModule(module).apply { for (source in compiledData.values) { - appendLine("// FILE: ${source.sourceFile.removePrefix("/")}") + appendLine("// FILE: ${File(source.sourceFile).name}") appendLine(source.smap ?: "") } } diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt index eca80223238..2da82e565d0 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/IrBackendInput.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.test.backend.ir +import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.backend.jvm.JvmIrCodegenFactory import org.jetbrains.kotlin.codegen.CodegenFactory import org.jetbrains.kotlin.codegen.state.GenerationState @@ -35,7 +36,8 @@ sealed class IrBackendInput : ResultingArtifact.BackendInput() { data class JvmIrBackendInput( val state: GenerationState, val codegenFactory: JvmIrCodegenFactory, - val backendInput: JvmIrCodegenFactory.JvmIrBackendInput + val backendInput: JvmIrCodegenFactory.JvmIrBackendInput, + val sourceFiles: List ) : IrBackendInput() { override val irModuleFragment: IrModuleFragment get() = backendInput.irModuleFragment diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/JvmIrBackendFacade.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/JvmIrBackendFacade.kt index ecb79ba0989..a81e79e5334 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/JvmIrBackendFacade.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/ir/JvmIrBackendFacade.kt @@ -5,6 +5,7 @@ package org.jetbrains.kotlin.test.backend.ir +import org.jetbrains.kotlin.KtPsiSourceFile import org.jetbrains.kotlin.backend.common.BackendException import org.jetbrains.kotlin.backend.jvm.MultifileFacadeFileEntry import org.jetbrains.kotlin.backend.jvm.lower.getFileClassInfoFromIrFile @@ -21,7 +22,6 @@ import org.jetbrains.kotlin.test.model.SourceFileInfo import org.jetbrains.kotlin.test.model.TestModule import org.jetbrains.kotlin.test.services.TestServices import org.jetbrains.kotlin.test.services.compilerConfigurationProvider -import java.io.File class JvmIrBackendFacade( testServices: TestServices @@ -48,14 +48,20 @@ class JvmIrBackendFacade( val configuration = testServices.compilerConfigurationProvider.getCompilerConfiguration(module) javaCompilerFacade.compileJavaFiles(module, configuration, state.factory) - fun sourceFileInfos(irFile: IrFile, allowNestedMultifileFacades: Boolean): List> = + fun sourceFileInfos(irFile: IrFile, allowNestedMultifileFacades: Boolean): List = when (val fileEntry = irFile.fileEntry) { is PsiIrFileEntry -> { - listOf(SourceFileInfo(fileEntry.psiFile, JvmFileClassUtil.getFileClassInfoNoResolve(fileEntry.psiFile as KtFile))) + listOf( + SourceFileInfo( + KtPsiSourceFile(fileEntry.psiFile), + JvmFileClassUtil.getFileClassInfoNoResolve(fileEntry.psiFile as KtFile) + ) + ) } is NaiveSourceBasedFileEntryImpl -> { - val file = File(fileEntry.name) - listOf(SourceFileInfo(file, getFileClassInfoFromIrFile(irFile, file.name))) + val sourceFile = inputArtifact.sourceFiles.find { it.path == fileEntry.name } + if (sourceFile == null) emptyList() // synthetic files, like CoroutineHelpers.kt, are ignored here + else listOf(SourceFileInfo(sourceFile, getFileClassInfoFromIrFile(irFile, sourceFile.name))) } is MultifileFacadeFileEntry -> { if (!allowNestedMultifileFacades) error("nested multi-file facades are not allowed") diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontend2IrConverter.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontend2IrConverter.kt index 95edb453b3f..6821f0c83d6 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontend2IrConverter.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/classic/ClassicFrontend2IrConverter.kt @@ -64,7 +64,8 @@ class ClassicFrontend2IrConverter( return IrBackendInput.JvmIrBackendInput( state, codegenFactory, - codegenFactory.convertToIr(CodegenFactory.IrConversionInput.fromGenerationStateAndFiles(state, psiFiles.values)) + codegenFactory.convertToIr(CodegenFactory.IrConversionInput.fromGenerationStateAndFiles(state, psiFiles.values)), + emptyList() ) } diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/Fir2IrResultsConverter.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/Fir2IrResultsConverter.kt index e00e416f65e..2c6d7a8d344 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/Fir2IrResultsConverter.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/Fir2IrResultsConverter.kt @@ -50,6 +50,7 @@ class Fir2IrResultsConverter( // TODO: handle fir from light tree val ktFiles = inputArtifact.firFiles.values.mapNotNull { it.psi as KtFile? } + val sourceFiles = inputArtifact.firFiles.values.mapNotNull { it.sourceFile } // Create and initialize the module and its dependencies val project = compilerConfigurationProvider.getProject(module) @@ -82,7 +83,8 @@ class Fir2IrResultsConverter( extensions, FirJvmBackendExtension(inputArtifact.session, components), notifyCodegenStart = {}, - ) + ), + sourceFiles ) } } diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFrontendFacade.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFrontendFacade.kt index aaedb253515..99d3e52da45 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFrontendFacade.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirFrontendFacade.kt @@ -18,7 +18,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.VfsBasedProjectEnvironment import org.jetbrains.kotlin.cli.jvm.config.jvmClasspathRoots import org.jetbrains.kotlin.cli.jvm.config.jvmModularRoots import org.jetbrains.kotlin.config.JVMConfigurationKeys -import org.jetbrains.kotlin.fir.analysis.FirAnalyzerFacade +import org.jetbrains.kotlin.fir.FirAnalyzerFacade import org.jetbrains.kotlin.fir.checkers.registerExtendedCommonCheckers import org.jetbrains.kotlin.fir.extensions.FirExtensionRegistrar import org.jetbrains.kotlin.fir.moduleData diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirOutputArtifact.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirOutputArtifact.kt index 7ba52fe1c27..bb7d2fb87b7 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirOutputArtifact.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/frontend/fir/FirOutputArtifact.kt @@ -5,9 +5,9 @@ package org.jetbrains.kotlin.test.frontend.fir +import org.jetbrains.kotlin.fir.AbstractFirAnalyzerFacade +import org.jetbrains.kotlin.fir.FirAnalyzerFacade import org.jetbrains.kotlin.fir.FirSession -import org.jetbrains.kotlin.fir.analysis.AbstractFirAnalyzerFacade -import org.jetbrains.kotlin.fir.analysis.FirAnalyzerFacade import org.jetbrains.kotlin.fir.declarations.FirFile import org.jetbrains.kotlin.test.model.FrontendKinds import org.jetbrains.kotlin.test.model.ResultingArtifact diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/model/ResultingArtifacts.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/model/ResultingArtifacts.kt index 3fc0ed18f43..ae8d30b52c8 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/model/ResultingArtifacts.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/model/ResultingArtifacts.kt @@ -5,19 +5,20 @@ package org.jetbrains.kotlin.test.model +import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.codegen.ClassFileFactory import org.jetbrains.kotlin.fileClasses.JvmFileClassInfo import org.jetbrains.kotlin.ir.backend.js.CompilerResult import org.jetbrains.kotlin.js.facade.TranslationResult import java.io.File -class SourceFileInfo( - val file: F, +class SourceFileInfo( + val sourceFile: KtSourceFile, val info: JvmFileClassInfo ) object BinaryArtifacts { - class Jvm(val classFileFactory: ClassFileFactory, val fileInfos: Collection>) : ResultingArtifact.Binary() { + class Jvm(val classFileFactory: ClassFileFactory, val fileInfos: Collection) : ResultingArtifact.Binary() { override val kind: BinaryKind get() = ArtifactKinds.Jvm } diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/services/sourceProviders/MainFunctionForBlackBoxTestsSourceProvider.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/services/sourceProviders/MainFunctionForBlackBoxTestsSourceProvider.kt index 4ef570b4a30..59d91ea7ea2 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/services/sourceProviders/MainFunctionForBlackBoxTestsSourceProvider.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/services/sourceProviders/MainFunctionForBlackBoxTestsSourceProvider.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.test.services.sourceProviders import com.intellij.psi.PsiFile +import org.jetbrains.kotlin.KtPsiSourceFile +import org.jetbrains.kotlin.KtSourceFile import org.jetbrains.kotlin.test.directives.CodegenTestDirectives.REQUIRES_SEPARATE_PROCESS import org.jetbrains.kotlin.test.directives.JvmEnvironmentConfigurationDirectives.JDK_KIND import org.jetbrains.kotlin.test.directives.model.RegisteredDirectives @@ -39,11 +41,12 @@ open class MainFunctionForBlackBoxTestsSourceProvider(testServices: TestServices return containsBoxMethod(file.originalContent) } - fun fileContainsBoxMethod(file: T): Boolean = - when (file) { - is PsiFile -> containsBoxMethod(file.text) - is File -> containsBoxMethod(file.readText()) - else -> error("Unknown file type: $file") + fun fileContainsBoxMethod(sourceFile: KtSourceFile): Boolean = + when (sourceFile) { + is KtPsiSourceFile -> containsBoxMethod(sourceFile.psiFile.text) + else -> with(sourceFile.getContentsAsStream().reader(Charsets.UTF_8)) { + containsBoxMethod(this.readText()) + } } fun containsBoxMethod(fileContent: String): Boolean { diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/SMAPTestUtil.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/SMAPTestUtil.kt index 96c7594f489..ad601fb261e 100644 --- a/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/SMAPTestUtil.kt +++ b/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/SMAPTestUtil.kt @@ -25,6 +25,7 @@ import org.jetbrains.kotlin.codegen.inline.GENERATE_SMAP import org.jetbrains.kotlin.test.KotlinBaseTest import org.jetbrains.kotlin.test.util.JUnit4Assertions import org.junit.Assert +import java.io.File import java.io.StringReader object SMAPTestUtil { @@ -62,7 +63,7 @@ object SMAPTestUtil { }.associateBy { it.sourceFile } for (source in sourceData) { - val ktFileName = "/" + source.sourceFile + val ktFileName = "/" + File(source.sourceFile).name .replace(".smap-nonseparate-compilation", ".kt") .replace(".smap-separate-compilation", ".kt") .replace(".smap", ".kt") diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/fir/AbstractFirBaseDiagnosticsTest.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/fir/AbstractFirBaseDiagnosticsTest.kt index a64eb0651f8..2839725ea8c 100644 --- a/compiler/tests-common/tests/org/jetbrains/kotlin/fir/AbstractFirBaseDiagnosticsTest.kt +++ b/compiler/tests-common/tests/org/jetbrains/kotlin/fir/AbstractFirBaseDiagnosticsTest.kt @@ -11,6 +11,7 @@ import com.intellij.openapi.vfs.VirtualFileManager import com.intellij.psi.PsiElement import com.intellij.psi.PsiElementFinder import com.intellij.psi.search.GlobalSearchScope +import org.jetbrains.kotlin.KtInMemoryTextSourceFile import org.jetbrains.kotlin.analyzer.ModuleInfo import org.jetbrains.kotlin.asJava.finder.JavaElementFinder import org.jetbrains.kotlin.checkers.BaseDiagnosticsTest @@ -46,6 +47,7 @@ import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.resolve.AnalyzingUtils import org.jetbrains.kotlin.resolve.PlatformDependentAnalyzerServices import org.jetbrains.kotlin.resolve.jvm.platform.JvmPlatformAnalyzerServices +import org.jetbrains.kotlin.toSourceLinesMapping import java.io.File abstract class AbstractFirBaseDiagnosticsTest : BaseDiagnosticsTest() { @@ -133,7 +135,12 @@ abstract class AbstractFirBaseDiagnosticsTest : BaseDiagnosticsTest() { if (useLightTree) { val lightTreeBuilder = LightTree2Fir(session, firProvider.kotlinScopeProvider) ktFiles.mapTo(firFiles) { - val firFile = lightTreeBuilder.buildFirFile(it.text, it.name, it.virtualFilePath) + val firFile = + lightTreeBuilder.buildFirFile( + it.text, + KtInMemoryTextSourceFile(it.name, it.virtualFilePath, it.text), + it.text.toSourceLinesMapping() + ) (session.firProvider as FirProviderImpl).recordFile(firFile) firFile } diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/fir/FirResolveBench.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/fir/FirResolveBench.kt index 9f28f4a4dbb..790006c8e4f 100644 --- a/compiler/tests-common/tests/org/jetbrains/kotlin/fir/FirResolveBench.kt +++ b/compiler/tests-common/tests/org/jetbrains/kotlin/fir/FirResolveBench.kt @@ -9,6 +9,7 @@ import com.intellij.openapi.util.io.FileUtil import com.intellij.openapi.util.text.StringUtil import com.intellij.openapi.vfs.CharsetToolkit import com.intellij.psi.PsiElement +import org.jetbrains.kotlin.KtIoFileSourceFile import org.jetbrains.kotlin.fir.builder.RawFirBuilder import org.jetbrains.kotlin.fir.declarations.FirFile import org.jetbrains.kotlin.fir.diagnostics.ConeStubDiagnostic @@ -26,6 +27,7 @@ import org.jetbrains.kotlin.fir.types.* import org.jetbrains.kotlin.fir.visitors.FirDefaultVisitorVoid import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.psi.psiUtil.startOffset +import org.jetbrains.kotlin.readSourceFileWithMapping import org.jetbrains.kotlin.utils.addToStdlib.sumByLong import java.io.File import java.io.PrintStream @@ -139,16 +141,18 @@ class FirResolveBench(val withProgress: Boolean, val listener: BenchListener? = return files.map { file -> val before = vmStateSnapshot() val firFile: FirFile - val code: String val time = measureNanoTime { - code = FileUtil.loadFile(file, CharsetToolkit.UTF8, true).trim() - firFile = builder.buildFirFile(code, file.name, file.path) + val sourceFile = KtIoFileSourceFile(file) + val (code, linesMapping) = with(file.inputStream().reader(Charsets.UTF_8)) { + this.readSourceFileWithMapping() + } + totalLines += linesMapping.linesCount + firFile = builder.buildFirFile(code, sourceFile, linesMapping) (builder.session.firProvider as FirProviderImpl).recordFile(firFile) } val after = vmStateSnapshot() val diff = after - before recordTime(builder::class, diff, time) - totalLines += StringUtil.countNewLines(code) firFile }.also { listener?.after(builder::class) diff --git a/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt b/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt index ada3ea7902b..6349f634afc 100644 --- a/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt +++ b/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt @@ -29,7 +29,7 @@ import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.JVMConfigurationKeys import org.jetbrains.kotlin.config.languageVersionSettings -import org.jetbrains.kotlin.fir.analysis.FirAnalyzerFacade +import org.jetbrains.kotlin.fir.FirAnalyzerFacade import org.jetbrains.kotlin.fir.backend.jvm.FirJvmBackendClassResolver import org.jetbrains.kotlin.fir.backend.jvm.FirJvmBackendExtension import org.jetbrains.kotlin.fir.createSessionForTests diff --git a/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/analysis/FirAnalyzerFacade.kt b/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/fir/FirAnalyzerFacade.kt similarity index 94% rename from compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/analysis/FirAnalyzerFacade.kt rename to compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/fir/FirAnalyzerFacade.kt index b0642f5c40c..f9af763d3ec 100644 --- a/compiler/fir/entrypoint/src/org/jetbrains/kotlin/fir/analysis/FirAnalyzerFacade.kt +++ b/compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/fir/FirAnalyzerFacade.kt @@ -1,17 +1,16 @@ /* - * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.fir.analysis +package org.jetbrains.kotlin.fir import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension import org.jetbrains.kotlin.backend.jvm.serialization.JvmIdSignatureDescriptor import org.jetbrains.kotlin.config.LanguageVersionSettings -import org.jetbrains.kotlin.fir.FirSession -import org.jetbrains.kotlin.fir.analysis.collectors.FirDiagnosticsCollector import org.jetbrains.kotlin.diagnostics.DiagnosticReporterFactory import org.jetbrains.kotlin.diagnostics.KtDiagnostic +import org.jetbrains.kotlin.fir.analysis.collectors.FirDiagnosticsCollector import org.jetbrains.kotlin.fir.backend.Fir2IrConverter import org.jetbrains.kotlin.fir.backend.Fir2IrResult import org.jetbrains.kotlin.fir.backend.jvm.Fir2IrJvmSpecialAnnotationSymbolProvider @@ -21,8 +20,6 @@ import org.jetbrains.kotlin.fir.builder.PsiHandlingMode import org.jetbrains.kotlin.fir.builder.RawFirBuilder import org.jetbrains.kotlin.fir.declarations.FirFile import org.jetbrains.kotlin.fir.lightTree.LightTree2Fir -import org.jetbrains.kotlin.fir.lightTree.LightTreeFile -import org.jetbrains.kotlin.fir.moduleData import org.jetbrains.kotlin.fir.resolve.ScopeSession import org.jetbrains.kotlin.fir.resolve.providers.firProvider import org.jetbrains.kotlin.fir.resolve.providers.impl.FirProviderImpl @@ -31,6 +28,7 @@ import org.jetbrains.kotlin.ir.backend.jvm.serialization.JvmDescriptorMangler import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.psi2ir.generators.GeneratorExtensions +import org.jetbrains.kotlin.sourceFiles.LightTreeFile abstract class AbstractFirAnalyzerFacade { abstract val scopeSession: ScopeSession @@ -63,7 +61,7 @@ class FirAnalyzerFacade( firFiles = if (useLightTree) { val builder = LightTree2Fir(session, firProvider.kotlinScopeProvider) lightTreeFiles.map { - builder.buildFirFile(it).also { firFile -> + builder.buildFirFile(it.lightTree, it.sourceFile, it.linesMapping).also { firFile -> firProvider.recordFile(firFile) } }