Support separate modules in compiler

Unless the compatibility option "-Xsingle-module" is passed, the compiler will
create two modules instead of one now (see TopDownAnalyzerFacadeForJVM): the
main module contains Kotlin and Java sources and binaries from the previous
compilation of the given module chunk, the dependency module contains all other
Kotlin and Java binaries. This fixes some issues where the compiler couldn't
detect that the used symbol was from another module, and did not forbid some
usages which are only possible inside the module (see KT-10001).

The ideal way to deal with modules here would be to exactly recreate the
project structure, for example as it's done in JvmAnalyzerFacade and usages.
This is postponed until later

 #KT-10001 Fixed
 #KT-11840 In Progress

compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2JVMCompilerArguments.java

@@ -95,6 +95,9 @@ public class K2JVMCompilerArguments extends CommonCompilerArguments {
     @Argument(value = "Xload-script-configs", description = "Load script configuration files from project directory tree")
     public boolean loadScriptConfigs;
 
+    @Argument(value = "Xsingle-module", description = "Combine modules for source files and binary dependencies into a single module")
+    public boolean singleModule;
+
     // Paths to output directories for friend modules.
     public String[] friendPaths;
 

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/K2JVMCompiler.kt

@@ -366,6 +366,7 @@ class K2JVMCompiler : CLICompiler<K2JVMCompilerArguments>() {
             configuration.put(CLIConfigurationKeys.ALLOW_KOTLIN_PACKAGE, arguments.allowKotlinPackage)
             configuration.put(CLIConfigurationKeys.REPORT_PERF, arguments.reportPerf)
             configuration.put(JVMConfigurationKeys.LOAD_SCRIPT_CONFIGS, arguments.loadScriptConfigs)
+            configuration.put(JVMConfigurationKeys.USE_SINGLE_MODULE, arguments.singleModule)
 
             arguments.declarationsOutputPath?.let { configuration.put(JVMConfigurationKeys.DECLARATIONS_JSON_PATH, it) }
         }

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmCliVirtualFileFinder.kt

@@ -17,17 +17,21 @@
 package org.jetbrains.kotlin.cli.jvm.compiler
 
 import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.psi.search.GlobalSearchScope
 import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClassFinder
 import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.utils.addToStdlib.check
 
-class JvmCliVirtualFileFinder(private val index: JvmDependenciesIndex) : VirtualFileKotlinClassFinder() {
-
+class JvmCliVirtualFileFinder(
+        private val index: JvmDependenciesIndex,
+        private val scope: GlobalSearchScope
+) : VirtualFileKotlinClassFinder() {
     override fun findVirtualFileWithHeader(classId: ClassId): VirtualFile? {
         val classFileName = classId.relativeClassName.asString().replace('.', '$')
         return index.findClass(classId, acceptedRootTypes = JavaRoot.OnlyBinary) { dir, rootType ->
             dir.findChild("$classFileName.class")?.let {
                 if (it.isValid) it else null
             }
-        }
+        }?.check { it in scope }
     }
 }

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmCliVirtualFileFinderFactory.kt

@@ -20,6 +20,7 @@ import com.intellij.psi.search.GlobalSearchScope
 import org.jetbrains.kotlin.load.kotlin.JvmVirtualFileFinder
 import org.jetbrains.kotlin.load.kotlin.JvmVirtualFileFinderFactory
 
+// TODO: create different JvmDependenciesIndex instances for different sets of source roots to improve performance
 class JvmCliVirtualFileFinderFactory(private val index: JvmDependenciesIndex) : JvmVirtualFileFinderFactory {
-    override fun create(scope: GlobalSearchScope): JvmVirtualFileFinder = JvmCliVirtualFileFinder(index)
+    override fun create(scope: GlobalSearchScope): JvmVirtualFileFinder = JvmCliVirtualFileFinder(index, scope)
 }

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCliJavaFileManagerImpl.kt

@@ -30,14 +30,14 @@ import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.resolve.jvm.KotlinCliJavaFileManager
 import org.jetbrains.kotlin.util.PerformanceCounter
-import java.util.ArrayList
+import org.jetbrains.kotlin.utils.addToStdlib.check
+import java.util.*
 import kotlin.properties.Delegates
 
-class KotlinCliJavaFileManagerImpl(private val myPsiManager: PsiManager)
-: CoreJavaFileManager(myPsiManager), KotlinCliJavaFileManager {
-
+class KotlinCliJavaFileManagerImpl(private val myPsiManager: PsiManager) : CoreJavaFileManager(myPsiManager), KotlinCliJavaFileManager {
     private val perfCounter = PerformanceCounter.create("Find Java class")
     private var index: JvmDependenciesIndex by Delegates.notNull()
+    private val allScope = GlobalSearchScope.allScope(myPsiManager.project)
 
     fun initIndex(packagesCache: JvmDependenciesIndex) {
         this.index = packagesCache
@@ -47,8 +47,8 @@ class KotlinCliJavaFileManagerImpl(private val myPsiManager: PsiManager)
         return perfCounter.time {
             val classNameWithInnerClasses = classId.relativeClassName.asString()
             index.findClass(classId) { dir, type ->
-                findClassGivenPackage(searchScope, dir, classNameWithInnerClasses, type)
-            }
+                findClassGivenPackage(allScope, dir, classNameWithInnerClasses, type)
+            }?.check { it.containingFile.virtualFile in searchScope }
         }
     }
 

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt

@@ -260,7 +260,7 @@ class KotlinCoreEnvironment private constructor(
 
     private fun findLocalDirectory(root: JvmContentRoot): VirtualFile? {
         val path = root.file
-        val localFile = applicationEnvironment.localFileSystem.findFileByPath(path.absolutePath)
+        val localFile = findLocalDirectory(path.absolutePath)
         if (localFile == null) {
             report(WARNING, "Classpath entry points to a non-existent location: $path")
             return null
@@ -268,6 +268,9 @@ class KotlinCoreEnvironment private constructor(
         return localFile
     }
 
+    internal fun findLocalDirectory(absolutePath: String): VirtualFile? =
+            applicationEnvironment.localFileSystem.findFileByPath(absolutePath)
+
     private fun findJarRoot(root: JvmClasspathRoot): VirtualFile? {
         val path = root.file
         val jarFile = applicationEnvironment.jarFileSystem.findFileByPath("${path}${URLUtil.JAR_SEPARATOR}")

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinToJVMBytecodeCompiler.kt

@@ -16,9 +16,14 @@
 
 package org.jetbrains.kotlin.cli.jvm.compiler
 
+import com.intellij.openapi.project.Project
 import com.intellij.openapi.util.io.JarUtil
+import com.intellij.openapi.vfs.VfsUtilCore
+import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.psi.PsiManager
 import com.intellij.psi.impl.PsiModificationTrackerImpl
+import com.intellij.psi.search.DelegatingGlobalSearchScope
+import com.intellij.psi.search.GlobalSearchScope
 import org.jetbrains.annotations.TestOnly
 import org.jetbrains.kotlin.analyzer.AnalysisResult
 import org.jetbrains.kotlin.asJava.FilteredJvmDiagnostics
@@ -402,14 +407,24 @@ object KotlinToJVMBytecodeCompiler {
         val analyzerWithCompilerReport = AnalyzerWithCompilerReport(collector)
         analyzerWithCompilerReport.analyzeAndReport(
                 environment.getSourceFiles(), object : AnalyzerWithCompilerReport.Analyzer {
-            override fun analyze(): AnalysisResult =
-                    TopDownAnalyzerFacadeForJVM.analyzeFilesWithJavaIntegration(
-                            environment.project,
-                            environment.getSourceFiles(),
-                            CliLightClassGenerationSupport.NoScopeRecordCliBindingTrace(),
-                            environment.configuration,
-                            { scope -> JvmPackagePartProvider(environment, scope) }
-                    )
+            override fun analyze(): AnalysisResult {
+                val project = environment.project
+                val moduleOutputs = environment.configuration.get(JVMConfigurationKeys.MODULES)?.mapNotNull { module ->
+                    environment.findLocalDirectory(module.getOutputDirectory())
+                }.orEmpty()
+                val sourcesOnly = TopDownAnalyzerFacadeForJVM.newModuleSearchScope(project, environment.getSourceFiles())
+                // To support partial and incremental compilation, we add the scope which contains binaries from output directories
+                // of the compiled modules (.class) to the list of scopes of the source module
+                val scope = if (moduleOutputs.isEmpty()) sourcesOnly else sourcesOnly.uniteWith(DirectoriesScope(project, moduleOutputs))
+                return TopDownAnalyzerFacadeForJVM.analyzeFilesWithJavaIntegration(
+                        project,
+                        environment.getSourceFiles(),
+                        CliLightClassGenerationSupport.NoScopeRecordCliBindingTrace(),
+                        environment.configuration,
+                        { scope -> JvmPackagePartProvider(environment, scope) },
+                        scope
+                )
+            }
 
             override fun reportEnvironmentErrors() {
                 reportRuntimeConflicts(collector, environment.configuration.jvmClasspathRoots)
@@ -436,6 +451,16 @@ object KotlinToJVMBytecodeCompiler {
             null
     }
 
+    class DirectoriesScope(
+            project: Project, private val directories: List<VirtualFile>
+    ) : DelegatingGlobalSearchScope(GlobalSearchScope.allScope(project)) {
+        // TODO: optimize somehow?
+        override fun contains(file: VirtualFile) =
+                directories.any { directory -> VfsUtilCore.isAncestor(directory, file, false) }
+
+        override fun toString() = "All files under: $directories"
+    }
+
     private fun generate(
             environment: KotlinCoreEnvironment,
             configuration: CompilerConfiguration,