Support separate modules in compiler

Unless the compatibility option "-Xsingle-module" is passed, the compiler will
create two modules instead of one now (see TopDownAnalyzerFacadeForJVM): the
main module contains Kotlin and Java sources and binaries from the previous
compilation of the given module chunk, the dependency module contains all other
Kotlin and Java binaries. This fixes some issues where the compiler couldn't
detect that the used symbol was from another module, and did not forbid some
usages which are only possible inside the module (see KT-10001).

The ideal way to deal with modules here would be to exactly recreate the
project structure, for example as it's done in JvmAnalyzerFacade and usages.
This is postponed until later

 #KT-10001 Fixed
 #KT-11840 In Progress
This commit is contained in:
Alexander Udalov
2016-09-27 18:50:33 +03:00
parent 9d778711d2
commit 45190d9453
15 changed files with 182 additions and 52 deletions
@@ -95,6 +95,9 @@ public class K2JVMCompilerArguments extends CommonCompilerArguments {
@Argument(value = "Xload-script-configs", description = "Load script configuration files from project directory tree")
public boolean loadScriptConfigs;
@Argument(value = "Xsingle-module", description = "Combine modules for source files and binary dependencies into a single module")
public boolean singleModule;
// Paths to output directories for friend modules.
public String[] friendPaths;
@@ -366,6 +366,7 @@ class K2JVMCompiler : CLICompiler<K2JVMCompilerArguments>() {
configuration.put(CLIConfigurationKeys.ALLOW_KOTLIN_PACKAGE, arguments.allowKotlinPackage)
configuration.put(CLIConfigurationKeys.REPORT_PERF, arguments.reportPerf)
configuration.put(JVMConfigurationKeys.LOAD_SCRIPT_CONFIGS, arguments.loadScriptConfigs)
configuration.put(JVMConfigurationKeys.USE_SINGLE_MODULE, arguments.singleModule)
arguments.declarationsOutputPath?.let { configuration.put(JVMConfigurationKeys.DECLARATIONS_JSON_PATH, it) }
}
@@ -17,17 +17,21 @@
package org.jetbrains.kotlin.cli.jvm.compiler
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.psi.search.GlobalSearchScope
import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClassFinder
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.utils.addToStdlib.check
class JvmCliVirtualFileFinder(private val index: JvmDependenciesIndex) : VirtualFileKotlinClassFinder() {
class JvmCliVirtualFileFinder(
private val index: JvmDependenciesIndex,
private val scope: GlobalSearchScope
) : VirtualFileKotlinClassFinder() {
override fun findVirtualFileWithHeader(classId: ClassId): VirtualFile? {
val classFileName = classId.relativeClassName.asString().replace('.', '$')
return index.findClass(classId, acceptedRootTypes = JavaRoot.OnlyBinary) { dir, rootType ->
dir.findChild("$classFileName.class")?.let {
if (it.isValid) it else null
}
}
}?.check { it in scope }
}
}
@@ -20,6 +20,7 @@ import com.intellij.psi.search.GlobalSearchScope
import org.jetbrains.kotlin.load.kotlin.JvmVirtualFileFinder
import org.jetbrains.kotlin.load.kotlin.JvmVirtualFileFinderFactory
// TODO: create different JvmDependenciesIndex instances for different sets of source roots to improve performance
class JvmCliVirtualFileFinderFactory(private val index: JvmDependenciesIndex) : JvmVirtualFileFinderFactory {
override fun create(scope: GlobalSearchScope): JvmVirtualFileFinder = JvmCliVirtualFileFinder(index)
override fun create(scope: GlobalSearchScope): JvmVirtualFileFinder = JvmCliVirtualFileFinder(index, scope)
}
@@ -30,14 +30,14 @@ import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.resolve.jvm.KotlinCliJavaFileManager
import org.jetbrains.kotlin.util.PerformanceCounter
import java.util.ArrayList
import org.jetbrains.kotlin.utils.addToStdlib.check
import java.util.*
import kotlin.properties.Delegates
class KotlinCliJavaFileManagerImpl(private val myPsiManager: PsiManager)
: CoreJavaFileManager(myPsiManager), KotlinCliJavaFileManager {
class KotlinCliJavaFileManagerImpl(private val myPsiManager: PsiManager) : CoreJavaFileManager(myPsiManager), KotlinCliJavaFileManager {
private val perfCounter = PerformanceCounter.create("Find Java class")
private var index: JvmDependenciesIndex by Delegates.notNull()
private val allScope = GlobalSearchScope.allScope(myPsiManager.project)
fun initIndex(packagesCache: JvmDependenciesIndex) {
this.index = packagesCache
@@ -47,8 +47,8 @@ class KotlinCliJavaFileManagerImpl(private val myPsiManager: PsiManager)
return perfCounter.time {
val classNameWithInnerClasses = classId.relativeClassName.asString()
index.findClass(classId) { dir, type ->
findClassGivenPackage(searchScope, dir, classNameWithInnerClasses, type)
}
findClassGivenPackage(allScope, dir, classNameWithInnerClasses, type)
}?.check { it.containingFile.virtualFile in searchScope }
}
}
@@ -260,7 +260,7 @@ class KotlinCoreEnvironment private constructor(
private fun findLocalDirectory(root: JvmContentRoot): VirtualFile? {
val path = root.file
val localFile = applicationEnvironment.localFileSystem.findFileByPath(path.absolutePath)
val localFile = findLocalDirectory(path.absolutePath)
if (localFile == null) {
report(WARNING, "Classpath entry points to a non-existent location: $path")
return null
@@ -268,6 +268,9 @@ class KotlinCoreEnvironment private constructor(
return localFile
}
internal fun findLocalDirectory(absolutePath: String): VirtualFile? =
applicationEnvironment.localFileSystem.findFileByPath(absolutePath)
private fun findJarRoot(root: JvmClasspathRoot): VirtualFile? {
val path = root.file
val jarFile = applicationEnvironment.jarFileSystem.findFileByPath("${path}${URLUtil.JAR_SEPARATOR}")
@@ -16,9 +16,14 @@
package org.jetbrains.kotlin.cli.jvm.compiler
import com.intellij.openapi.project.Project
import com.intellij.openapi.util.io.JarUtil
import com.intellij.openapi.vfs.VfsUtilCore
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.psi.PsiManager
import com.intellij.psi.impl.PsiModificationTrackerImpl
import com.intellij.psi.search.DelegatingGlobalSearchScope
import com.intellij.psi.search.GlobalSearchScope
import org.jetbrains.annotations.TestOnly
import org.jetbrains.kotlin.analyzer.AnalysisResult
import org.jetbrains.kotlin.asJava.FilteredJvmDiagnostics
@@ -402,14 +407,24 @@ object KotlinToJVMBytecodeCompiler {
val analyzerWithCompilerReport = AnalyzerWithCompilerReport(collector)
analyzerWithCompilerReport.analyzeAndReport(
environment.getSourceFiles(), object : AnalyzerWithCompilerReport.Analyzer {
override fun analyze(): AnalysisResult =
TopDownAnalyzerFacadeForJVM.analyzeFilesWithJavaIntegration(
environment.project,
environment.getSourceFiles(),
CliLightClassGenerationSupport.NoScopeRecordCliBindingTrace(),
environment.configuration,
{ scope -> JvmPackagePartProvider(environment, scope) }
)
override fun analyze(): AnalysisResult {
val project = environment.project
val moduleOutputs = environment.configuration.get(JVMConfigurationKeys.MODULES)?.mapNotNull { module ->
environment.findLocalDirectory(module.getOutputDirectory())
}.orEmpty()
val sourcesOnly = TopDownAnalyzerFacadeForJVM.newModuleSearchScope(project, environment.getSourceFiles())
// To support partial and incremental compilation, we add the scope which contains binaries from output directories
// of the compiled modules (.class) to the list of scopes of the source module
val scope = if (moduleOutputs.isEmpty()) sourcesOnly else sourcesOnly.uniteWith(DirectoriesScope(project, moduleOutputs))
return TopDownAnalyzerFacadeForJVM.analyzeFilesWithJavaIntegration(
project,
environment.getSourceFiles(),
CliLightClassGenerationSupport.NoScopeRecordCliBindingTrace(),
environment.configuration,
{ scope -> JvmPackagePartProvider(environment, scope) },
scope
)
}
override fun reportEnvironmentErrors() {
reportRuntimeConflicts(collector, environment.configuration.jvmClasspathRoots)
@@ -436,6 +451,16 @@ object KotlinToJVMBytecodeCompiler {
null
}
class DirectoriesScope(
project: Project, private val directories: List<VirtualFile>
) : DelegatingGlobalSearchScope(GlobalSearchScope.allScope(project)) {
// TODO: optimize somehow?
override fun contains(file: VirtualFile) =
directories.any { directory -> VfsUtilCore.isAncestor(directory, file, false) }
override fun toString() = "All files under: $directories"
}
private fun generate(
environment: KotlinCoreEnvironment,
configuration: CompilerConfiguration,
@@ -56,6 +56,9 @@ public class JVMConfigurationKeys {
public static final CompilerConfigurationKey<Boolean> USE_TYPE_TABLE =
CompilerConfigurationKey.create("use type table in serializer");
public static final CompilerConfigurationKey<Boolean> USE_SINGLE_MODULE =
CompilerConfigurationKey.create("combine modules for source files and binary dependencies into a single module");
public static final CompilerConfigurationKey<JvmTarget> JVM_TARGET =
CompilerConfigurationKey.create("JVM bytecode target version");
@@ -118,9 +118,7 @@ fun createContainerForTopDownAnalyzerForJvm(
): ComponentProvider = createContainerForLazyResolveWithJava(
moduleContext, bindingTrace, declarationProviderFactory, moduleContentScope, moduleClassResolver,
CompilerEnvironment, lookupTracker, packagePartProvider, languageVersionSettings, useLazyResolve = false
).apply {
initJvmBuiltInsForTopDownAnalysis(moduleContext.module, languageVersionSettings)
}
)
fun createContainerForTopDownSingleModuleAnalyzerForJvm(
@@ -134,6 +132,7 @@ fun createContainerForTopDownSingleModuleAnalyzerForJvm(
moduleContext, bindingTrace, declarationProviderFactory, moduleContentScope,
LookupTracker.DO_NOTHING, packagePartProvider, languageVersionSettings, SingleModuleClassResolver()
).apply {
initJvmBuiltInsForTopDownAnalysis(moduleContext.module, languageVersionSettings)
get<SingleModuleClassResolver>().resolver = get<JavaDescriptorResolver>()
}
@@ -16,7 +16,10 @@
package org.jetbrains.kotlin.resolve.jvm
import com.intellij.ide.highlighter.JavaFileType
import com.intellij.openapi.project.Project
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.psi.search.DelegatingGlobalSearchScope
import com.intellij.psi.search.GlobalSearchScope
import org.jetbrains.kotlin.analyzer.AnalysisResult
import org.jetbrains.kotlin.config.CommonConfigurationKeys
@@ -27,13 +30,17 @@ import org.jetbrains.kotlin.container.get
import org.jetbrains.kotlin.context.ContextForNewModule
import org.jetbrains.kotlin.context.MutableModuleContext
import org.jetbrains.kotlin.context.ProjectContext
import org.jetbrains.kotlin.descriptors.ClassDescriptor
import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
import org.jetbrains.kotlin.descriptors.PackagePartProvider
import org.jetbrains.kotlin.descriptors.impl.CompositePackageFragmentProvider
import org.jetbrains.kotlin.descriptors.impl.ModuleDependenciesImpl
import org.jetbrains.kotlin.frontend.java.di.createContainerForTopDownAnalyzerForJvm
import org.jetbrains.kotlin.frontend.java.di.initJvmBuiltInsForTopDownAnalysis
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.load.java.lazy.ModuleClassResolverImpl
import org.jetbrains.kotlin.load.java.lazy.ModuleClassResolver
import org.jetbrains.kotlin.load.java.structure.JavaClass
import org.jetbrains.kotlin.load.java.structure.impl.JavaClassImpl
import org.jetbrains.kotlin.load.kotlin.DeserializationComponentsForJava
import org.jetbrains.kotlin.load.kotlin.incremental.IncrementalPackageFragmentProvider
import org.jetbrains.kotlin.load.kotlin.incremental.IncrementalPackagePartProvider
@@ -48,19 +55,23 @@ import org.jetbrains.kotlin.resolve.jvm.extensions.AnalysisCompletedHandlerExten
import org.jetbrains.kotlin.resolve.jvm.extensions.PackageFragmentProviderExtension
import org.jetbrains.kotlin.resolve.jvm.platform.JvmPlatform
import org.jetbrains.kotlin.resolve.lazy.KotlinCodeAnalyzer
import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactory
import org.jetbrains.kotlin.resolve.lazy.declarations.FileBasedDeclarationProviderFactory
import java.util.*
object TopDownAnalyzerFacadeForJVM {
@JvmStatic
@JvmOverloads
fun analyzeFilesWithJavaIntegration(
project: Project,
files: Collection<KtFile>,
trace: BindingTrace,
configuration: CompilerConfiguration,
packagePartProviderFactory: (GlobalSearchScope) -> PackagePartProvider
packagePartProviderFactory: (GlobalSearchScope) -> PackagePartProvider,
sourceModuleSearchScope: GlobalSearchScope = newModuleSearchScope(project, files)
): AnalysisResult {
val moduleContext = TopDownAnalyzerFacadeForJVM.createContextWithSealedModule(project, configuration)
val moduleContext = createModuleContext(project, configuration)
val storageManager = moduleContext.storageManager
val module = moduleContext.module
@@ -68,29 +79,51 @@ object TopDownAnalyzerFacadeForJVM {
val lookupTracker = incrementalComponents?.getLookupTracker() ?: LookupTracker.DO_NOTHING
val targetIds = configuration.get(JVMConfigurationKeys.MODULES)?.map(::TargetId)
val resolverByClass = object : (JavaClass) -> JavaDescriptorResolver {
lateinit var resolver: JavaDescriptorResolver
val separateModules = !configuration.getBoolean(JVMConfigurationKeys.USE_SINGLE_MODULE)
override fun invoke(javaClass: JavaClass): JavaDescriptorResolver {
return resolver
}
val sourceScope = if (separateModules) sourceModuleSearchScope else GlobalSearchScope.allScope(project)
val moduleClassResolver = SourceOrBinaryModuleClassResolver(sourceScope)
val languageVersionSettings =
configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS, LanguageVersionSettingsImpl.DEFAULT)
val dependencyModule = if (separateModules) {
val dependenciesContext = ContextForNewModule(
moduleContext, Name.special("<dependencies of ${configuration.getNotNull(CommonConfigurationKeys.MODULE_NAME)}>"),
JvmPlatform, module.builtIns
)
// Scope for the dependency module contains everything except files present in the scope for the source module
val dependencyScope = GlobalSearchScope.notScope(sourceScope)
val dependenciesContainer = createContainerForTopDownAnalyzerForJvm(
dependenciesContext, trace, DeclarationProviderFactory.EMPTY, dependencyScope, lookupTracker,
packagePartProviderFactory(dependencyScope), languageVersionSettings, moduleClassResolver
)
moduleClassResolver.compiledCodeResolver = dependenciesContainer.get<JavaDescriptorResolver>()
dependenciesContext.setDependencies(dependenciesContext.module, dependenciesContext.module.builtIns.builtInsModule)
dependenciesContext.initializeModuleContents(moduleClassResolver.compiledCodeResolver.packageFragmentProvider)
dependenciesContext.module
}
else null
val sourceScope = GlobalSearchScope.allScope(project)
// Note that it's necessary to create container for sources _after_ creation of container for dependencies because
// CliLightClassGenerationSupport#initialize is invoked when container is created, so only the last module descriptor is going
// to be stored in CliLightClassGenerationSupport, and it better be the source one (otherwise light classes would not be found)
// TODO: get rid of duplicate invocation of CodeAnalyzerInitializer#initialize, or refactor CliLightClassGenerationSupport
val container = createContainerForTopDownAnalyzerForJvm(
moduleContext,
trace,
FileBasedDeclarationProviderFactory(storageManager, files),
sourceScope,
lookupTracker,
moduleContext, trace, FileBasedDeclarationProviderFactory(storageManager, files), sourceScope, lookupTracker,
IncrementalPackagePartProvider.create(
packagePartProviderFactory(sourceScope), targetIds, incrementalComponents, storageManager
),
configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS, LanguageVersionSettingsImpl.DEFAULT),
ModuleClassResolverImpl(resolverByClass)
)
resolverByClass.resolver = container.get<JavaDescriptorResolver>()
languageVersionSettings, moduleClassResolver
).apply {
initJvmBuiltInsForTopDownAnalysis(module, languageVersionSettings)
}
moduleClassResolver.sourceCodeResolver = container.get<JavaDescriptorResolver>()
val additionalProviders = ArrayList<PackageFragmentProvider>()
if (incrementalComponents != null) {
@@ -104,10 +137,16 @@ object TopDownAnalyzerFacadeForJVM {
additionalProviders.add(container.get<JavaDescriptorResolver>().packageFragmentProvider)
// TODO: consider putting extension package fragment providers into the dependency module
PackageFragmentProviderExtension.getInstances(project).mapNotNullTo(additionalProviders) { extension ->
extension.getPackageFragmentProvider(project, module, storageManager, trace, null)
}
// TODO: remove dependencyModule from friends
module.setDependencies(ModuleDependenciesImpl(
listOfNotNull(module, dependencyModule, module.builtIns.builtInsModule),
if (dependencyModule != null) setOf(dependencyModule) else emptySet()
))
module.initialize(CompositePackageFragmentProvider(
listOf(container.get<KotlinCodeAnalyzer>().packageFragmentProvider) +
additionalProviders
@@ -123,14 +162,46 @@ object TopDownAnalyzerFacadeForJVM {
return AnalysisResult.success(trace.bindingContext, module)
}
fun createContextWithSealedModule(project: Project, configuration: CompilerConfiguration): MutableModuleContext {
fun newModuleSearchScope(project: Project, files: Collection<KtFile>): GlobalSearchScope {
// In case of separate modules, the source module scope generally consists of the following scopes:
// 1) scope which only contains passed Kotlin source files (.kt and .kts)
// 2) scope which contains all Java source files (.java) in the project
return GlobalSearchScope.filesScope(project, files.map { it.virtualFile }.toSet()).uniteWith(AllJavaSourcesInProjectScope(project))
}
// TODO: limit this scope to the Java source roots, which the module has in its CONTENT_ROOTS
class AllJavaSourcesInProjectScope(project: Project) : DelegatingGlobalSearchScope(GlobalSearchScope.allScope(project)) {
// 'isDirectory' check is needed because otherwise directories such as 'frontend.java' would be recognized
// as Java source files, which makes no sense
override fun contains(file: VirtualFile) =
file.fileType === JavaFileType.INSTANCE && !file.isDirectory
override fun toString() = "All Java sources in the project"
}
class SourceOrBinaryModuleClassResolver(private val sourceScope: GlobalSearchScope) : ModuleClassResolver {
lateinit var compiledCodeResolver: JavaDescriptorResolver
lateinit var sourceCodeResolver: JavaDescriptorResolver
override fun resolveClass(javaClass: JavaClass): ClassDescriptor? {
val resolver = if (javaClass is JavaClassImpl && javaClass.psi.containingFile.virtualFile in sourceScope)
sourceCodeResolver
else
compiledCodeResolver
return resolver.resolveClass(javaClass)
}
}
fun createContextWithSealedModule(project: Project, configuration: CompilerConfiguration): MutableModuleContext =
createModuleContext(project, configuration).apply {
setDependencies(module, module.builtIns.builtInsModule)
}
private fun createModuleContext(project: Project, configuration: CompilerConfiguration): MutableModuleContext {
val projectContext = ProjectContext(project)
val builtIns = JvmBuiltIns(projectContext.storageManager)
val context = ContextForNewModule(
return ContextForNewModule(
projectContext, Name.special("<${configuration.getNotNull(CommonConfigurationKeys.MODULE_NAME)}>"),
JvmPlatform, builtIns
JvmPlatform, JvmBuiltIns(projectContext.storageManager)
)
context.setDependencies(context.module, builtIns.builtInsModule)
return context
}
}
+1
View File
@@ -9,6 +9,7 @@ where advanced options include:
-Xskip-metadata-version-check Try loading binary incompatible classes, may cause crashes
-Xdump-declarations-to <path> Path to JSON file to dump Java to Kotlin declaration mappings
-Xload-script-configs Load script configuration files from project directory tree
-Xsingle-module Combine modules for source files and binary dependencies into a single module
-Xno-inline Disable method inlining
-Xrepeat <count> Repeat compilation (for performance analysis)
-Xplugin <path> Load plugins from the given classpath
@@ -0,0 +1,9 @@
fun foo1(p: Pair<Int?, Int>): Int {
if (p.first != null) return p.first!!
return p.second
}
fun foo2(p: Pair<Int?, Int>): Int {
if (p.first != null) return <!SMARTCAST_IMPOSSIBLE!>p.first<!>
return p.second
}
@@ -0,0 +1,4 @@
package
public fun foo1(/*0*/ p: kotlin.Pair<kotlin.Int?, kotlin.Int>): kotlin.Int
public fun foo2(/*0*/ p: kotlin.Pair<kotlin.Int?, kotlin.Int>): kotlin.Int
@@ -1054,6 +1054,12 @@ public class DiagnosticsTestWithStdLibGenerated extends AbstractDiagnosticsTestW
doTest(fileName);
}
@TestMetadata("kt10001.kt")
public void testKt10001() throws Exception {
String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/testsWithStdLib/regression/kt10001.kt");
doTest(fileName);
}
@TestMetadata("kt2082.kt")
public void testKt2082() throws Exception {
String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/diagnostics/testsWithStdLib/regression/kt2082.kt");
@@ -4774,18 +4774,18 @@ public class IntentionTestGenerated extends AbstractIntentionTest {
doTest(fileName);
}
@TestMetadata("withComposedModifiers.kt")
public void testWithComposedModifiers() throws Exception {
String fileName = KotlinTestUtils.navigationMetadata("idea/testData/intentions/convertSecondaryConstructorToPrimary/withComposedModifiers.kt");
doTest(fileName);
}
@TestMetadata("withComments.kt")
public void testWithComments() throws Exception {
String fileName = KotlinTestUtils.navigationMetadata("idea/testData/intentions/convertSecondaryConstructorToPrimary/withComments.kt");
doTest(fileName);
}
@TestMetadata("withComposedModifiers.kt")
public void testWithComposedModifiers() throws Exception {
String fileName = KotlinTestUtils.navigationMetadata("idea/testData/intentions/convertSecondaryConstructorToPrimary/withComposedModifiers.kt");
doTest(fileName);
}
@TestMetadata("withDelegation.kt")
public void testWithDelegation() throws Exception {
String fileName = KotlinTestUtils.navigationMetadata("idea/testData/intentions/convertSecondaryConstructorToPrimary/withDelegation.kt");