From 8ccd7c7d12191916507015b300795e04a682b333 Mon Sep 17 00:00:00 2001 From: Dmitriy Novozhilov Date: Tue, 14 Feb 2023 17:58:20 +0200 Subject: [PATCH] [FIR] Create context and scope for incremental compilation separately Scope for incremental compilation refers to binaries from previous step of IC. It is used not only in IC context itself, but also it is subtracted from original libraries scope. Before previous commit there was such scheme: 1. create incremental compilation context for files of specific session 2. subtract IC scope from main libraries scope 3. use updated libraries scope to create library session 4. create all needed source session(s) So here was a side effect of creating new IC context, which 1. is smelling code, because it increases mind complexity 2. hard to implement with new session utilities So to fix this problem this commit changes the scheme above: 1. create IC scope and modify libraries scope 2. create libraries session 3. create source session(s) and IC context for them --- .../cli/common/FirSessionConstructionUtils.kt | 72 +++++++------------ .../jvm/compiler/pipeline/compilerPipeline.kt | 57 +++++++++------ .../cli/metadata/FirMetadataSerializer.kt | 20 +++--- .../plugin/impl/ScriptJvmCompilerImpls.kt | 22 +++--- 4 files changed, 86 insertions(+), 85 deletions(-) diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/common/FirSessionConstructionUtils.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/common/FirSessionConstructionUtils.kt index 1db2f25a283..a7e9b6d7c8e 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/common/FirSessionConstructionUtils.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/common/FirSessionConstructionUtils.kt @@ -19,8 +19,6 @@ import org.jetbrains.kotlin.fir.java.FirProjectSessionProvider import org.jetbrains.kotlin.fir.session.* import org.jetbrains.kotlin.fir.session.environment.AbstractProjectEnvironment import org.jetbrains.kotlin.fir.session.environment.AbstractProjectFileSearchScope -import org.jetbrains.kotlin.incremental.components.EnumWhenTracker -import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.js.resolve.JsPlatformAnalyzerServices import org.jetbrains.kotlin.library.metadata.resolver.KotlinResolvedLibrary import org.jetbrains.kotlin.load.kotlin.PackageAndMetadataPartProvider @@ -67,20 +65,9 @@ fun prepareJvmSessions( ): List> { val javaSourcesScope = projectEnvironment.getSearchScopeForProjectJavaSources() - class Context( - val javaSourcesScope: AbstractProjectFileSearchScope, - val lookupTracker: LookupTracker?, - val enumWhenTracker: EnumWhenTracker? - ) - return prepareSessions( files, configuration, rootModuleName, JvmPlatforms.unspecifiedJvmPlatform, JvmPlatformAnalyzerServices, libraryList, isCommonSource, fileBelongsToModule, - createContext = { - val lookupTracker = configuration.get(CommonConfigurationKeys.LOOKUP_TRACKER) - val enumWhenTracker = configuration.get(CommonConfigurationKeys.ENUM_WHEN_TRACKER) - Context(javaSourcesScope, lookupTracker, enumWhenTracker) - }, createLibrarySession = { sessionProvider -> FirJvmSessionFactory.createLibrarySession( Name.identifier(rootModuleName), @@ -94,17 +81,17 @@ fun prepareJvmSessions( ) } - ) { context, moduleFiles, moduleData, sessionProvider, sessionConfigurator -> + ) { moduleFiles, moduleData, sessionProvider, sessionConfigurator -> FirJvmSessionFactory.createModuleBasedSession( moduleData, sessionProvider, - context.javaSourcesScope, + javaSourcesScope, projectEnvironment, createProviderAndScopeForIncrementalCompilation(moduleFiles), extensionRegistrars, configuration.languageVersionSettings, - context.lookupTracker, - context.enumWhenTracker, + configuration.get(CommonConfigurationKeys.LOOKUP_TRACKER), + configuration.get(CommonConfigurationKeys.ENUM_WHEN_TRACKER), needRegisterJavaElementFinder = true, registerExtraComponents = {}, sessionConfigurator, @@ -132,7 +119,6 @@ fun prepareJsSessions( return prepareSessions( files, configuration, rootModuleName, JsPlatforms.defaultJsPlatform, JsPlatformAnalyzerServices, libraryList, isCommonSource, fileBelongsToModule, - createContext = { null }, createLibrarySession = { sessionProvider -> FirJsSessionFactory.createLibrarySession( Name.identifier(rootModuleName), @@ -143,7 +129,7 @@ fun prepareJsSessions( registerExtraComponents = {}, ) } - ) { _, _, moduleData, sessionProvider, sessionConfigurator -> + ) { _, moduleData, sessionProvider, sessionConfigurator -> FirJsSessionFactory.createModuleBasedSession( moduleData, sessionProvider, @@ -175,8 +161,7 @@ fun prepareNativeSessions( ): List> { return prepareSessions( files, configuration, rootModuleName, NativePlatforms.unspecifiedNativePlatform, NativePlatformAnalyzerServices, - libraryList, isCommonSource, fileBelongsToModule, createContext = { null }, - createLibrarySession = { sessionProvider -> + libraryList, isCommonSource, fileBelongsToModule, createLibrarySession = { sessionProvider -> FirNativeSessionFactory.createLibrarySession( Name.identifier(rootModuleName), resolvedLibraries, @@ -186,7 +171,7 @@ fun prepareNativeSessions( registerExtraComponents = {}, ) } - ) { _, _, moduleData, sessionProvider, sessionConfigurator -> + ) { _, moduleData, sessionProvider, sessionConfigurator -> FirNativeSessionFactory.createModuleBasedSession( moduleData, sessionProvider, @@ -218,8 +203,7 @@ fun prepareCommonSessions( ): List> { return prepareSessions( files, configuration, rootModuleName, CommonPlatforms.defaultCommonPlatform, CommonPlatformAnalyzerServices, - libraryList, isCommonSource, fileBelongsToModule, createContext = { null }, - createLibrarySession = { sessionProvider -> + libraryList, isCommonSource, fileBelongsToModule, createLibrarySession = { sessionProvider -> FirCommonSessionFactory.createLibrarySession( Name.identifier(rootModuleName), sessionProvider, @@ -231,7 +215,7 @@ fun prepareCommonSessions( registerExtraComponents = {}, ) } - ) { _, moduleFiles, moduleData, sessionProvider, sessionConfigurator -> + ) { moduleFiles, moduleData, sessionProvider, sessionConfigurator -> FirCommonSessionFactory.createModuleBasedSession( moduleData, sessionProvider, @@ -251,9 +235,9 @@ fun prepareCommonSessions( // ---------------------------------------------------- Implementation ---------------------------------------------------- -private typealias FirSessionProducer = (C, List, FirModuleData, FirProjectSessionProvider, FirSessionConfigurator.() -> Unit) -> FirSession +private typealias FirSessionProducer = (List, FirModuleData, FirProjectSessionProvider, FirSessionConfigurator.() -> Unit) -> FirSession -private inline fun prepareSessions( +private inline fun prepareSessions( files: List, configuration: CompilerConfiguration, rootModuleName: String, @@ -262,9 +246,8 @@ private inline fun prepareSessions( libraryList: DependencyListForCliModule, isCommonSource: (F) -> Boolean, fileBelongsToModule: (F, String) -> Boolean, - createContext: () -> C, createLibrarySession: (FirProjectSessionProvider) -> FirSession, - createSourceSession: FirSessionProducer, + createSourceSession: FirSessionProducer, ): List> { val languageVersionSettings = configuration.languageVersionSettings @@ -272,8 +255,6 @@ private inline fun prepareSessions( val hmppModuleStructure = configuration.get(CommonConfigurationKeys.HMPP_MODULE_STRUCTURE) val sessionProvider = FirProjectSessionProvider() - val context = createContext() - createLibrarySession(sessionProvider) val extendedAnalysisMode = configuration.getBoolean(CommonConfigurationKeys.USE_FIR_EXTENDED_CHECKERS) val sessionConfigurator: FirSessionConfigurator.() -> Unit = { @@ -285,25 +266,24 @@ private inline fun prepareSessions( return when { !isMppEnabled -> listOf( createSessionForNonMppProject( - context, files, rootModuleName, libraryList, targetPlatform, analyzerServices, + files, rootModuleName, libraryList, targetPlatform, analyzerServices, sessionProvider, sessionConfigurator, createSourceSession ) ) hmppModuleStructure == null -> createSessionsForLegacyMppProject( - context, files, rootModuleName, libraryList, targetPlatform, analyzerServices, + files, rootModuleName, libraryList, targetPlatform, analyzerServices, sessionProvider, sessionConfigurator, isCommonSource, createSourceSession ) else -> createSessionsForHmppProject( - context, files, hmppModuleStructure, libraryList, targetPlatform, analyzerServices, + files, hmppModuleStructure, libraryList, targetPlatform, analyzerServices, sessionProvider, sessionConfigurator, fileBelongsToModule, createSourceSession ) } } -private inline fun createSessionForNonMppProject( - context: C, +private inline fun createSessionForNonMppProject( files: List, rootModuleName: String, libraryList: DependencyListForCliModule, @@ -311,7 +291,7 @@ private inline fun createSessionForNonMppProject( analyzerServices: PlatformDependentAnalyzerServices, sessionProvider: FirProjectSessionProvider, noinline sessionConfigurator: FirSessionConfigurator.() -> Unit, - createFirSession: FirSessionProducer, + createFirSession: FirSessionProducer, ): SessionWithSources { val platformModuleData = FirModuleDataImpl( Name.identifier(rootModuleName), @@ -322,12 +302,11 @@ private inline fun createSessionForNonMppProject( analyzerServices ) - val session = createFirSession(context, files, platformModuleData, sessionProvider, sessionConfigurator) + val session = createFirSession(files, platformModuleData, sessionProvider, sessionConfigurator) return SessionWithSources(session, files) } -private inline fun createSessionsForLegacyMppProject( - context: C, +private inline fun createSessionsForLegacyMppProject( files: List, rootModuleName: String, libraryList: DependencyListForCliModule, @@ -336,7 +315,7 @@ private inline fun createSessionsForLegacyMppProject( sessionProvider: FirProjectSessionProvider, noinline sessionConfigurator: FirSessionConfigurator.() -> Unit, isCommonSource: (F) -> Boolean, - createFirSession: FirSessionProducer, + createFirSession: FirSessionProducer, ): List> { val commonModuleData = FirModuleDataImpl( Name.identifier("${rootModuleName}-common"), @@ -362,8 +341,8 @@ private inline fun createSessionsForLegacyMppProject( (if (isCommonSource(file)) commonFiles else platformFiles).add(file) } - val commonSession = createFirSession(context, commonFiles, commonModuleData, sessionProvider, sessionConfigurator) - val platformSession = createFirSession(context, platformFiles, platformModuleData, sessionProvider, sessionConfigurator) + val commonSession = createFirSession(commonFiles, commonModuleData, sessionProvider, sessionConfigurator) + val platformSession = createFirSession(platformFiles, platformModuleData, sessionProvider, sessionConfigurator) return listOf( SessionWithSources(commonSession, commonFiles), @@ -371,8 +350,7 @@ private inline fun createSessionsForLegacyMppProject( ) } -private inline fun createSessionsForHmppProject( - context: C, +private inline fun createSessionsForHmppProject( files: List, hmppModuleStructure: HmppCliModuleStructure, libraryList: DependencyListForCliModule, @@ -381,7 +359,7 @@ private inline fun createSessionsForHmppProject( sessionProvider: FirProjectSessionProvider, noinline sessionConfigurator: FirSessionConfigurator.() -> Unit, fileBelongsToModule: (F, String) -> Boolean, - createFirSession: FirSessionProducer, + createFirSession: FirSessionProducer, ): List> { val moduleDataForHmppModule = LinkedHashMap() @@ -403,7 +381,7 @@ private inline fun createSessionsForHmppProject( return hmppModuleStructure.modules.map { module -> val moduleData = moduleDataForHmppModule.getValue(module) val sources = files.filter { fileBelongsToModule(it, module.name) } - val session = createFirSession(context, sources, moduleData, sessionProvider, sessionConfigurator) + val session = createFirSession(sources, moduleData, sessionProvider, sessionConfigurator) SessionWithSources(session, sources) } } diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipeline.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipeline.kt index 4a449b6d9f4..1c55f24319c 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipeline.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/pipeline/compilerPipeline.kt @@ -309,6 +309,12 @@ fun compileModuleToAnalyzedFir( var librariesScope = projectEnvironment.getSearchScopeForProjectLibraries() val rootModuleName = input.targetId.name + val incrementalCompilationScope = createIncrementalCompilationScope( + moduleConfiguration, + projectEnvironment, + incrementalExcludesScope + )?.also { librariesScope -= it } + val extensionRegistrars = (projectEnvironment as? VfsBasedProjectEnvironment) ?.let { FirExtensionRegistrar.getInstances(it.project) } ?: emptyList() @@ -331,10 +337,8 @@ fun compileModuleToAnalyzedFir( projectEnvironment, scope, previousStepsSymbolProviders, - incrementalExcludesScope - )?.also { (_, _, precompiledBinariesFileScope) -> - precompiledBinariesFileScope?.let { librariesScope -= it } - } + incrementalCompilationScope + ) } ) @@ -390,30 +394,43 @@ fun writeOutputs( return true } +fun createIncrementalCompilationScope( + configuration: CompilerConfiguration, + projectEnvironment: AbstractProjectEnvironment, + incrementalExcludesScope: AbstractProjectFileSearchScope? +): AbstractProjectFileSearchScope? { + if (!needCreateIncrementalCompilationScope(configuration)) return null + val dir = configuration[JVMConfigurationKeys.OUTPUT_DIRECTORY] ?: return null + return projectEnvironment.getSearchScopeByDirectories(setOf(dir)).let { + if (incrementalExcludesScope?.isEmpty != false) it + else it - incrementalExcludesScope + } +} + +private fun needCreateIncrementalCompilationScope(configuration: CompilerConfiguration, ): Boolean { + if (configuration.get(JVMConfigurationKeys.MODULES) == null) return false + if (configuration.get(JVMConfigurationKeys.INCREMENTAL_COMPILATION_COMPONENTS) == null) return false + return true +} + fun createContextForIncrementalCompilation( - compilerConfiguration: CompilerConfiguration, + configuration: CompilerConfiguration, projectEnvironment: AbstractProjectEnvironment, sourceScope: AbstractProjectFileSearchScope, previousStepsSymbolProviders: List, - incrementalExcludesScope: AbstractProjectFileSearchScope?, + incrementalCompilationScope: AbstractProjectFileSearchScope? ): IncrementalCompilationContext? { - val targetIds = compilerConfiguration.get(JVMConfigurationKeys.MODULES)?.map(::TargetId) - val incrementalComponents = compilerConfiguration.get(JVMConfigurationKeys.INCREMENTAL_COMPILATION_COMPONENTS) - if (targetIds == null || incrementalComponents == null) return null - val incrementalCompilationScope = - compilerConfiguration[JVMConfigurationKeys.OUTPUT_DIRECTORY]?.let { dir -> - projectEnvironment.getSearchScopeByDirectories(setOf(dir)).let { - if (incrementalExcludesScope?.isEmpty != false) it - else it - incrementalExcludesScope - } - } + if (incrementalCompilationScope == null && previousStepsSymbolProviders.isEmpty()) return null + val targetIds = configuration.get(JVMConfigurationKeys.MODULES)?.map(::TargetId) ?: return null + val incrementalComponents = configuration.get(JVMConfigurationKeys.INCREMENTAL_COMPILATION_COMPONENTS) ?: return null - return if (incrementalCompilationScope == null && previousStepsSymbolProviders.isEmpty()) null - else IncrementalCompilationContext( - previousStepsSymbolProviders, IncrementalPackagePartProvider( + return IncrementalCompilationContext( + previousStepsSymbolProviders, + IncrementalPackagePartProvider( projectEnvironment.getPackagePartProvider(sourceScope), targetIds.map(incrementalComponents::getIncrementalCache) - ), incrementalCompilationScope + ), + incrementalCompilationScope ) } diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/FirMetadataSerializer.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/FirMetadataSerializer.kt index ccd14d690fa..f1f3f7481a2 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/FirMetadataSerializer.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/metadata/FirMetadataSerializer.kt @@ -15,6 +15,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.VfsBasedProjectEnvironment import org.jetbrains.kotlin.cli.jvm.compiler.createContextForIncrementalCompilation import org.jetbrains.kotlin.cli.jvm.compiler.pipeline.collectSources import org.jetbrains.kotlin.cli.jvm.compiler.pipeline.createContextForIncrementalCompilation +import org.jetbrains.kotlin.cli.jvm.compiler.pipeline.createIncrementalCompilationScope import org.jetbrains.kotlin.cli.jvm.compiler.toAbstractProjectEnvironment import org.jetbrains.kotlin.cli.jvm.config.jvmClasspathRoots import org.jetbrains.kotlin.cli.jvm.config.jvmModularRoots @@ -76,20 +77,23 @@ internal class FirMetadataSerializer( val groupedSources = collectSources(configuration, projectEnvironment, messageCollector) val extensionRegistrars = FirExtensionRegistrar.getInstances(projectEnvironment.project) val ltFiles = groupedSources.let { it.commonSources + it.platformSources }.toList() - val sourceScope = projectEnvironment.getSearchScopeBySourceFiles(ltFiles) - val providerAndScopeForIncrementalCompilation = createContextForIncrementalCompilation( + val incrementalCompilationScope = createIncrementalCompilationScope( configuration, projectEnvironment, - sourceScope, - previousStepsSymbolProviders = emptyList(), incrementalExcludesScope = null - )?.also { (_, _, precompiledBinariesFileScope) -> - precompiledBinariesFileScope?.let { librariesScope -= it } - } + )?.also { librariesScope -= it } val sessionsWithSources = prepareCommonSessions( ltFiles, configuration, projectEnvironment, rootModuleName, extensionRegistrars, librariesScope, libraryList, groupedSources.isCommonSourceForLt, groupedSources.fileBelongsToModuleForLt, - createProviderAndScopeForIncrementalCompilation = { providerAndScopeForIncrementalCompilation } + createProviderAndScopeForIncrementalCompilation = { files -> + createContextForIncrementalCompilation( + configuration, + projectEnvironment, + projectEnvironment.getSearchScopeBySourceFiles(files), + previousStepsSymbolProviders = emptyList(), + incrementalCompilationScope + ) + } ) sessionsWithSources.map { (session, files) -> val firFiles = session.buildFirViaLightTree(files, diagnosticsReporter, performanceManager::addSourcesStats) diff --git a/plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/impl/ScriptJvmCompilerImpls.kt b/plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/impl/ScriptJvmCompilerImpls.kt index 503d0d3a9fe..2099a709c54 100644 --- a/plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/impl/ScriptJvmCompilerImpls.kt +++ b/plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/impl/ScriptJvmCompilerImpls.kt @@ -339,18 +339,12 @@ private fun doCompileWithK2( val projectEnvironment = context.environment.toAbstractProjectEnvironment() val compilerEnvironment = ModuleCompilerEnvironment(projectEnvironment, diagnosticsReporter) - val sourcesScope = compilerEnvironment.projectEnvironment.getSearchScopeBySourceFiles(sources) var librariesScope = projectEnvironment.getSearchScopeForProjectLibraries() - val providerAndScopeForIncrementalCompilation = createContextForIncrementalCompilation( + val incrementalCompilationScope = createIncrementalCompilationScope( compilerInput.configuration, projectEnvironment, - sourcesScope, - emptyList(), - null - )?.also { (_, _, precompiledBinariesFileScope) -> - precompiledBinariesFileScope?.let { librariesScope -= it } - } - + incrementalExcludesScope = null + )?.also { librariesScope -= it } val extensionRegistrars = (projectEnvironment as? VfsBasedProjectEnvironment) ?.let { FirExtensionRegistrar.getInstances(it.project) } .orEmpty() @@ -364,7 +358,15 @@ private fun doCompileWithK2( val session = prepareJvmSessions( sourceFiles, kotlinCompilerConfiguration, projectEnvironment, rootModuleName, extensionRegistrars, librariesScope, libraryList, isCommonSourceForPsi, fileBelongsToModuleForPsi, - createProviderAndScopeForIncrementalCompilation = { providerAndScopeForIncrementalCompilation } + createProviderAndScopeForIncrementalCompilation = { files -> + createContextForIncrementalCompilation( + compilerInput.configuration, + projectEnvironment, + compilerEnvironment.projectEnvironment.getSearchScopeBySourceFiles(files.map { KtPsiSourceFile(it) }), + emptyList(), + incrementalCompilationScope + ) + } ).single().session session.scriptDefinitionProviderService?.run {