Introduce JavaModule, refactor module graph construction

#KT-18598 In Progress
 #KT-18599 In Progress

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/ClasspathRootsResolver.kt

@@ -19,9 +19,13 @@ package org.jetbrains.kotlin.cli.jvm.compiler
 import com.intellij.openapi.vfs.StandardFileSystems
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.openapi.vfs.VirtualFileManager
+import com.intellij.psi.PsiJavaModule
+import com.intellij.psi.PsiManager
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.*
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
+import org.jetbrains.kotlin.cli.common.messages.MessageUtil
 import org.jetbrains.kotlin.cli.jvm.config.JavaSourceRoot
 import org.jetbrains.kotlin.cli.jvm.config.JvmClasspathRoot
 import org.jetbrains.kotlin.cli.jvm.config.JvmContentRoot
@@ -30,10 +34,12 @@ import org.jetbrains.kotlin.cli.jvm.modules.CliJavaModuleFinder
 import org.jetbrains.kotlin.config.ContentRoot
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.isValidJavaFqName
+import org.jetbrains.kotlin.resolve.jvm.modules.JavaModule
 import org.jetbrains.kotlin.resolve.jvm.modules.JavaModuleGraph
 import org.jetbrains.kotlin.resolve.jvm.modules.JavaModuleInfo
 
 internal class ClasspathRootsResolver(
+        private val psiManager: PsiManager,
         private val messageCollector: MessageCollector?,
         private val contentRootToVirtualFile: (JvmContentRoot) -> VirtualFile?
 ) {
@@ -43,18 +49,26 @@ internal class ClasspathRootsResolver(
     fun convertClasspathRoots(contentRoots: Iterable<ContentRoot>): List<JavaRoot> {
         val result = mutableListOf<JavaRoot>()
 
+        val modules = ArrayList<JavaModule>()
+
         for (contentRoot in contentRoots) {
             if (contentRoot !is JvmContentRoot) continue
             val root = contentRootToVirtualFile(contentRoot) ?: continue
 
             when (contentRoot) {
                 is JavaSourceRoot -> {
-                    result += JavaRoot(root, JavaRoot.RootType.SOURCE, contentRoot.packagePrefix?.let { prefix ->
-                        if (isValidJavaFqName(prefix)) FqName(prefix)
-                        else null.also {
-                            report(STRONG_WARNING, "Invalid package prefix name is ignored: $prefix")
-                        }
-                    })
+                    val modularRoot = modularSourceRoot(root)
+                    if (modularRoot != null) {
+                        modules += modularRoot
+                    }
+                    else {
+                        result += JavaRoot(root, JavaRoot.RootType.SOURCE, contentRoot.packagePrefix?.let { prefix ->
+                            if (isValidJavaFqName(prefix)) FqName(prefix)
+                            else null.also {
+                                report(STRONG_WARNING, "Invalid package prefix name is ignored: $prefix")
+                            }
+                        })
+                    }
                 }
                 is JvmClasspathRoot -> {
                     result += JavaRoot(root, JavaRoot.RootType.BINARY)
@@ -63,43 +77,78 @@ internal class ClasspathRootsResolver(
             }
         }
 
-        addModularRoots(result)
+        addModularRoots(modules, result)
 
         return result
     }
 
-    private fun addModularRoots(result: MutableList<JavaRoot>) {
-        val jrtFileSystem = javaModuleFinder.jrtFileSystem ?: return
+    private fun modularSourceRoot(root: VirtualFile): JavaModule.Explicit? {
+        val moduleInfoFile =
+                when {
+                    root.isDirectory -> root.findChild(PsiJavaModule.MODULE_INFO_FILE)
+                    root.name == PsiJavaModule.MODULE_INFO_FILE -> root
+                    else -> null
+                } ?: return null
 
-        val rootModules = computeRootModules()
-        val allDependencies = javaModuleGraph.getAllDependencies(rootModules).also { modules ->
-            report(LOGGING, "Loading modules: $modules")
+        val psiFile = psiManager.findFile(moduleInfoFile) ?: return null
+        val psiJavaModule = psiFile.children.singleOrNull { it is PsiJavaModule } as? PsiJavaModule ?: return null
+        return JavaModule.Explicit(JavaModuleInfo.create(psiJavaModule), root, moduleInfoFile, isBinary = false)
+    }
+
+    private fun addModularRoots(modules: List<JavaModule>, result: MutableList<JavaRoot>) {
+        val sourceModules = modules.filterIsInstance<JavaModule.Explicit>().filterNot(JavaModule::isBinary)
+        if (sourceModules.size > 1) {
+            for (module in sourceModules) {
+                report(ERROR, "Too many source module declarations found", module.moduleInfoFile)
+            }
+            return
+        }
+
+        for (module in modules) {
+            // TODO: report a diagnostic if a module with this name was already added
+            javaModuleFinder.addUserModule(module)
+        }
+
+        if (javaModuleFinder.allObservableModules.none()) return
+
+        val rootModules =
+                if (sourceModules.isNotEmpty()) {
+                    // TODO: support an option similar to --add-modules
+                    listOf(sourceModules.single().name)
+                }
+                else computeDefaultRootModules() // TODO: + everything from module path
+
+        // TODO: if at least one automatic module is added, add all automatic modules as per java.lang.module javadoc
+        val allDependencies = javaModuleGraph.getAllDependencies(rootModules).also { loadedModules ->
+            report(LOGGING, "Loading modules: $loadedModules")
         }
 
         for (moduleName in allDependencies) {
-            // TODO: support modules not only from Java runtime image, but from a separate module path
-            val root = jrtFileSystem.findFileByPath("/modules/$moduleName")
-            if (root == null) {
+            val module = javaModuleFinder.findModule(moduleName)
+            if (module == null) {
                 report(ERROR, "Module $moduleName cannot be found in the module graph")
             }
             else {
-                result.add(JavaRoot(root, JavaRoot.RootType.BINARY))
+                result.add(JavaRoot(
+                        module.moduleRoot,
+                        if (module.isBinary) JavaRoot.RootType.BINARY else JavaRoot.RootType.SOURCE
+                ))
             }
         }
     }
 
     // See http://openjdk.java.net/jeps/261
-    private fun computeRootModules(): List<String> {
+    private fun computeDefaultRootModules(): List<String> {
         val result = arrayListOf<String>()
 
-        val systemModules = javaModuleFinder.computeAllSystemModules()
+        val systemModules = javaModuleFinder.systemModules.associateBy(JavaModule::name)
         val javaSeExists = "java.se" in systemModules
         if (javaSeExists) {
             // The java.se module is a root, if it exists.
             result.add("java.se")
         }
 
-        fun JavaModuleInfo.exportsAtLeastOnePackageUnqualified(): Boolean = exports.any { it.toModules.isEmpty() }
+        fun JavaModule.Explicit.exportsAtLeastOnePackageUnqualified(): Boolean = moduleInfo.exports.any { it.toModules.isEmpty() }
 
         if (!javaSeExists) {
             // If it does not exist then every java.* module on the upgrade module path or among the system modules
@@ -122,10 +171,13 @@ internal class ClasspathRootsResolver(
         return result
     }
 
-    private fun report(severity: CompilerMessageSeverity, message: String) {
+    private fun report(severity: CompilerMessageSeverity, message: String, file: VirtualFile? = null) {
         if (messageCollector == null) {
-            throw IllegalStateException("$severity: $message (no MessageCollector configured)")
+            throw IllegalStateException("${if (file != null) file.path + ":" else ""}$severity: $message (no MessageCollector configured)")
         }
-        messageCollector.report(severity, message)
+        messageCollector.report(
+                severity, message,
+                if (file == null) null else CompilerMessageLocation.create(MessageUtil.virtualFileToPath(file))
+        )
     }
 }

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt

@@ -49,6 +49,7 @@ import com.intellij.openapi.vfs.VirtualFileSystem
 import com.intellij.openapi.vfs.impl.ZipHandler
 import com.intellij.psi.FileContextProvider
 import com.intellij.psi.PsiElementFinder
+import com.intellij.psi.PsiManager
 import com.intellij.psi.augment.PsiAugmentProvider
 import com.intellij.psi.augment.TypeAnnotationModifier
 import com.intellij.psi.compiled.ClassFileDecompilers
@@ -191,7 +192,7 @@ class KotlinCoreEnvironment private constructor(
 
         val messageCollector = configuration.get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)
 
-        classpathRootsResolver = ClasspathRootsResolver(messageCollector, this::contentRootToVirtualFile)
+        classpathRootsResolver = ClasspathRootsResolver(PsiManager.getInstance(project), messageCollector, this::contentRootToVirtualFile)
 
         initialRoots = classpathRootsResolver.convertClasspathRoots(configuration.getList(JVMConfigurationKeys.CONTENT_ROOTS))
 

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/modules/CliJavaModuleFinder.kt

@@ -16,27 +16,38 @@
 
 package org.jetbrains.kotlin.cli.jvm.modules
 
+import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.openapi.vfs.VirtualFileSystem
 import com.intellij.psi.PsiJavaModule
+import org.jetbrains.kotlin.resolve.jvm.modules.JavaModule
 import org.jetbrains.kotlin.resolve.jvm.modules.JavaModuleFinder
 import org.jetbrains.kotlin.resolve.jvm.modules.JavaModuleInfo
 
-internal class CliJavaModuleFinder(val jrtFileSystem: VirtualFileSystem?) : JavaModuleFinder {
-    internal fun computeAllSystemModules(): Map<String, JavaModuleInfo> {
-        return jrtFileSystem?.findFileByPath("/modules")?.children.orEmpty()
-                .mapNotNull { root -> root.findChild(PsiJavaModule.MODULE_INFO_CLS_FILE) }
-                .mapNotNull((JavaModuleInfo)::read)
-                .associateBy { moduleInfo -> moduleInfo.moduleName }
+internal class CliJavaModuleFinder(private val jrtFileSystem: VirtualFileSystem?) : JavaModuleFinder {
+    private val userModules = linkedMapOf<String, JavaModule>()
+
+    /**
+     * @return true if the module was added successfully, false otherwise
+     */
+    fun addUserModule(module: JavaModule): Boolean {
+        if (module.name in userModules) return false
+        userModules[module.name] = module
+        return true
     }
 
-    override fun findModule(name: String): JavaModuleInfo? {
-        val file = jrtFileSystem?.findFileByPath("/modules/$name/${PsiJavaModule.MODULE_INFO_CLS_FILE}")
-        if (file != null) {
-            val moduleInfo = JavaModuleInfo.read(file)
-            if (moduleInfo != null) {
-                return moduleInfo
-            }
-        }
-        return null
+    val allObservableModules: Sequence<JavaModule>
+        get() = systemModules + userModules.values
+
+    val systemModules: Sequence<JavaModule.Explicit>
+        get() = jrtFileSystem?.findFileByPath("/modules")?.children.orEmpty().asSequence().mapNotNull(this::findSystemModule)
+
+    override fun findModule(name: String): JavaModule? =
+            jrtFileSystem?.findFileByPath("/modules/$name")?.let(this::findSystemModule)
+            ?: userModules[name]
+
+    private fun findSystemModule(moduleRoot: VirtualFile): JavaModule.Explicit? {
+        val file = moduleRoot.findChild(PsiJavaModule.MODULE_INFO_CLS_FILE) ?: return null
+        val moduleInfo = JavaModuleInfo.read(file) ?: return null
+        return JavaModule.Explicit(moduleInfo, moduleRoot, file, isBinary = true)
     }
 }

compiler/frontend.java/src/org/jetbrains/kotlin/resolve/jvm/modules/JavaModule.kt

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.resolve.jvm.modules
+
+import com.intellij.openapi.vfs.VirtualFile
+import org.jetbrains.kotlin.name.FqName
+
+interface JavaModule {
+    /**
+     * The name of the module. For explicit modules, this is the name specified in the module-info file.
+     * For automatic modules, this is the name in the Automatic-Module-Name attribute in the manifest,
+     * or an automatically inferred name in case of absence of such attribute.
+     */
+    val name: String
+
+    /**
+     * A directory or the root of a .jar file on the module path. In case of an explicit module,
+     * module-info.class file can be found in its children.
+     */
+    val moduleRoot: VirtualFile
+
+    /**
+     * A module-info.class or module-info.java file where this module was loaded from.
+     */
+    val moduleInfoFile: VirtualFile?
+
+    /**
+     * `true` if this module is either an automatic module on the module path, or an explicit module loaded from module-info.class.
+     * `false` if this module is an explicit module loaded from module-info.java.
+     */
+    val isBinary: Boolean
+
+    /**
+     * `true` if this module exports the package with the given FQ name to all dependent modules.
+     * For explicit modules, this means that there's an _unqualified_ (without the 'to' clause) exports statement in the module-info file.
+     * For automatic modules, this is always `true`.
+     *
+     * Note that it's assumed that the module contains this package.
+     */
+    fun exports(packageFqName: FqName): Boolean
+
+    /**
+     * `true` if this module exports the package with the given FQ name to a module with the given name.
+     * For explicit modules, this means that there's either an unqualified exports statement (without the 'to' clause)
+     * in the module-info file, or a _qualified_ statement (with the 'to' clause) with the given module name at the right hand side.
+     * For automatic modules, this is always `true`.
+     *
+     * Note that it's assumed that the module contains this package.
+     */
+    fun exportsTo(packageFqName: FqName, moduleName: String): Boolean
+
+    class Automatic(override val name: String, override val moduleRoot: VirtualFile) : JavaModule {
+        override val moduleInfoFile: VirtualFile? get() = null
+
+        override val isBinary: Boolean get() = true
+
+        override fun exports(packageFqName: FqName): Boolean = true
+
+        override fun exportsTo(packageFqName: FqName, moduleName: String): Boolean = true
+
+        override fun toString(): String = name
+    }
+
+    class Explicit(
+            val moduleInfo: JavaModuleInfo,
+            override val moduleRoot: VirtualFile,
+            override val moduleInfoFile: VirtualFile,
+            override val isBinary: Boolean
+    ) : JavaModule {
+        override val name: String
+            get() = moduleInfo.moduleName
+
+        override fun exports(packageFqName: FqName): Boolean {
+            return moduleInfo.exports.any { (fqName, toModules) ->
+                fqName == packageFqName && toModules.isEmpty()
+            }
+        }
+
+        override fun exportsTo(packageFqName: FqName, moduleName: String): Boolean {
+            return moduleInfo.exports.any { (fqName, toModules) ->
+                fqName == packageFqName && (toModules.isEmpty() || moduleName in toModules)
+            }
+        }
+
+        override fun toString(): String = name
+    }
+}

compiler/frontend.java/src/org/jetbrains/kotlin/resolve/jvm/modules/JavaModuleFinder.kt

@@ -17,5 +17,5 @@
 package org.jetbrains.kotlin.resolve.jvm.modules
 
 interface JavaModuleFinder {
-    fun findModule(name: String): JavaModuleInfo?
+    fun findModule(name: String): JavaModule?
 }

compiler/frontend.java/src/org/jetbrains/kotlin/resolve/jvm/modules/JavaModuleGraph.kt

@@ -19,19 +19,20 @@ package org.jetbrains.kotlin.resolve.jvm.modules
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
 
 class JavaModuleGraph(finder: JavaModuleFinder) {
-    private val moduleInfo: (String) -> JavaModuleInfo? =
+    private val module: (String) -> JavaModule? =
             LockBasedStorageManager.NO_LOCKS.createMemoizedFunctionWithNullableValues(finder::findModule)
 
     fun getAllDependencies(moduleNames: List<String>): List<String> {
-        // Every module implicitly depends on java.base
-        val visited = linkedSetOf("java.base")
+        val visited = moduleNames.toMutableSet()
 
-        fun dfs(module: String) {
-            if (!visited.add(module)) return
-            val moduleInfo = moduleInfo(module) ?: return
-            for ((moduleName, isTransitive) in moduleInfo.requires) {
-                if (isTransitive) {
-                    dfs(moduleName)
+        // Every module implicitly depends on java.base
+        visited += "java.base"
+
+        fun dfs(moduleName: String) {
+            val moduleInfo = (module(moduleName) as? JavaModule.Explicit)?.moduleInfo ?: return
+            for ((dependencyModuleName, isTransitive) in moduleInfo.requires) {
+                if (!visited.add(dependencyModuleName) && isTransitive) {
+                    dfs(dependencyModuleName)
                 }
             }
         }

compiler/frontend.java/src/org/jetbrains/kotlin/resolve/jvm/modules/JavaModuleInfo.kt

@@ -17,6 +17,8 @@
 package org.jetbrains.kotlin.resolve.jvm.modules
 
 import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.psi.PsiJavaModule
+import com.intellij.psi.PsiModifier
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.utils.compactIfPossible
 import org.jetbrains.org.objectweb.asm.ClassReader
@@ -39,6 +41,18 @@ class JavaModuleInfo(
             "Module $moduleName (${requires.size} requires, ${exports.size} exports)"
 
     companion object {
+        fun create(psiJavaModule: PsiJavaModule): JavaModuleInfo {
+            return JavaModuleInfo(
+                    psiJavaModule.name,
+                    psiJavaModule.requires.map { statement ->
+                        JavaModuleInfo.Requires(statement.moduleName!!, statement.hasModifierProperty(PsiModifier.TRANSITIVE))
+                    },
+                    psiJavaModule.exports.map { statement ->
+                        JavaModuleInfo.Exports(FqName(statement.packageName!!), statement.moduleNames)
+                    }
+            )
+        }
+
         fun read(file: VirtualFile): JavaModuleInfo? {
             val contents = try { file.contentsToByteArray() } catch (e: IOException) { return null }