Move calculating dependencies from script sources to plugin

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt

@@ -74,7 +74,6 @@ import org.jetbrains.kotlin.cli.common.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PRO
 import org.jetbrains.kotlin.cli.common.config.ContentRoot
 import org.jetbrains.kotlin.cli.common.config.KotlinSourceRoot
 import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots
-import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.STRONG_WARNING
@@ -112,7 +111,6 @@ import org.jetbrains.kotlin.resolve.jvm.extensions.PackageFragmentProviderExtens
 import org.jetbrains.kotlin.resolve.jvm.modules.JavaModuleResolver
 import org.jetbrains.kotlin.resolve.lazy.declarations.CliDeclarationProviderFactoryService
 import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactoryService
-import org.jetbrains.kotlin.resolve.multiplatform.isCommonSource
 import org.jetbrains.kotlin.utils.PathUtil
 import java.io.File
 import java.lang.reflect.Field
@@ -199,9 +197,7 @@ class KotlinCoreEnvironment private constructor(
 
         sourceFiles += createKtFiles(project)
 
-        val (newSourcesClasspath, newSources, _) = collectScriptsCompilationDependencies(configuration, project, sourceFiles)
+        collectAdditionalSources(project)
-        configuration.addJvmClasspathRoots(newSourcesClasspath)
-        sourceFiles += newSources
 
         sourceFiles.sortBy { it.virtualFile.path }
 
@@ -267,6 +263,32 @@ class KotlinCoreEnvironment private constructor(
         })
     }
 
+    private fun collectAdditionalSources(project: MockProject) {
+        var unprocessedSources: Collection<KtFile> = sourceFiles
+        val processedSources = HashSet<KtFile>()
+        val processedSourcesByExtension = HashMap<CollectAdditionalSourcesExtension, Collection<KtFile>>()
+        // repeat feeding extensions with sources while new sources a being added
+        var sourceCollectionIterations = 0
+        while (unprocessedSources.isNotEmpty()) {
+            if (sourceCollectionIterations++ > 10) { // TODO: consider using some appropriate global constant
+                throw IllegalStateException("Unable to collect additional sources in reasonable number of iterations")
+            }
+            processedSources.addAll(unprocessedSources)
+            val allNewSources = ArrayList<KtFile>()
+            for (extension in CollectAdditionalSourcesExtension.getInstances(project)) {
+                // do not feed the extension with the sources it returned on the previous iteration
+                val sourcesToProcess = unprocessedSources - (processedSourcesByExtension[extension] ?: emptyList())
+                val newSources = extension.collectAdditionalSourcesAndUpdateConfiguration(sourcesToProcess, configuration, project)
+                if (newSources.isNotEmpty()) {
+                    allNewSources.addAll(newSources)
+                    processedSourcesByExtension[extension] = newSources
+                }
+            }
+            unprocessedSources = allNewSources.filterNot { processedSources.contains(it) }.distinct()
+            sourceFiles += unprocessedSources
+        }
+    }
+
     fun createPackagePartProvider(scope: GlobalSearchScope): JvmPackagePartProvider {
         return JvmPackagePartProvider(configuration.languageVersionSettings, scope).apply {
             addRoots(initialRoots, configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY))
@@ -383,7 +405,7 @@ class KotlinCoreEnvironment private constructor(
     private fun createKtFiles(project: Project): List<KtFile> =
         createSourceFilesFromSourceRoots(configuration, project, getSourceRootsCheckingForDuplicates())
 
-    internal fun report(severity: CompilerMessageSeverity, message: String) = report(configuration, severity, message)
+    internal fun report(severity: CompilerMessageSeverity, message: String) = configuration.report(severity, message)
 
     companion object {
         private val LOG = Logger.getInstance(KotlinCoreEnvironment::class.java)
@@ -436,67 +458,6 @@ class KotlinCoreEnvironment private constructor(
         // used in the daemon for jar cache cleanup
         val applicationEnvironment: JavaCoreApplicationEnvironment? get() = ourApplicationEnvironment
 
-        internal fun report(
-            configuration: CompilerConfiguration,
-            severity: CompilerMessageSeverity,
-            message: String,
-            location: CompilerMessageLocation? = null
-        ) {
-            configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY).report(severity, message, location)
-        }
-
-        internal fun createSourceFilesFromSourceRoots(
-            configuration: CompilerConfiguration,
-            project: Project,
-            sourceRoots: List<KotlinSourceRoot>,
-            reportLocation: CompilerMessageLocation? = null
-        ): MutableList<KtFile> {
-            val localFileSystem = VirtualFileManager.getInstance().getFileSystem(StandardFileSystems.FILE_PROTOCOL)
-            val psiManager = PsiManager.getInstance(project)
-
-            val processedFiles = hashSetOf<VirtualFile>()
-            val result = mutableListOf<KtFile>()
-
-            val virtualFileCreator = PreprocessedFileCreator(project)
-
-            for ((sourceRootPath, isCommon) in sourceRoots) {
-                val vFile = localFileSystem.findFileByPath(sourceRootPath)
-                if (vFile == null) {
-                    val message = "Source file or directory not found: $sourceRootPath"
-
-                    val buildFilePath = configuration.get(JVMConfigurationKeys.MODULE_XML_FILE)
-                    if (buildFilePath != null && Logger.isInitialized()) {
-                        KotlinCoreEnvironment.LOG.warn("$message\n\nbuild file path: $buildFilePath\ncontent:\n${buildFilePath.readText()}")
-                    }
-
-                    report(configuration, ERROR, message, reportLocation)
-                    continue
-                }
-
-                if (!vFile.isDirectory && vFile.fileType != KotlinFileType.INSTANCE) {
-                    report(configuration, ERROR, "Source entry is not a Kotlin file: $sourceRootPath", reportLocation)
-                    continue
-                }
-
-                for (file in File(sourceRootPath).walkTopDown()) {
-                    if (!file.isFile) continue
-
-                    val virtualFile = localFileSystem.findFileByPath(file.absolutePath)?.let(virtualFileCreator::create)
-                    if (virtualFile != null && processedFiles.add(virtualFile)) {
-                        val psiFile = psiManager.findFile(virtualFile)
-                        if (psiFile is KtFile) {
-                            result.add(psiFile)
-                            if (isCommon) {
-                                psiFile.isCommonSource = true
-                            }
-                        }
-                    }
-                }
-            }
-
-            return result
-        }
-
         private fun getOrCreateApplicationEnvironmentForProduction(configuration: CompilerConfiguration): JavaCoreApplicationEnvironment {
             synchronized(APPLICATION_LOCK) {
                 if (ourApplicationEnvironment != null)
@@ -632,6 +593,7 @@ class KotlinCoreEnvironment private constructor(
             PreprocessedVirtualFileFactoryExtension.registerExtensionPoint(project)
             JsSyntheticTranslateExtension.registerExtensionPoint(project)
             CompilerConfigurationExtension.registerExtensionPoint(project)
+            CollectAdditionalSourcesExtension.registerExtensionPoint(project)
             IrGenerationExtension.registerExtensionPoint(project)
         }
 
@@ -728,8 +690,7 @@ class KotlinCoreEnvironment private constructor(
 
             if (!CoreJrtFileSystem.isModularJdk(javaRoot)) {
                 if (classesRoots.isEmpty()) {
-                    val messageCollector = get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)
+                    report(ERROR, "No class roots are found in the JDK path: $javaRoot")
-                    messageCollector?.report(ERROR, "No class roots are found in the JDK path: $javaRoot")
                 } else {
                     addJvmSdkRoots(classesRoots)
                 }

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/coreEnvironmentUtils.kt

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.cli.jvm.compiler
+
+import com.intellij.openapi.diagnostic.Logger
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.vfs.StandardFileSystems
+import com.intellij.openapi.vfs.VirtualFile
+import com.intellij.openapi.vfs.VirtualFileManager
+import com.intellij.psi.PsiManager
+import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
+import org.jetbrains.kotlin.cli.common.config.KotlinSourceRoot
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.config.JVMConfigurationKeys
+import org.jetbrains.kotlin.extensions.PreprocessedFileCreator
+import org.jetbrains.kotlin.idea.KotlinFileType
+import org.jetbrains.kotlin.psi.KtFile
+import org.jetbrains.kotlin.resolve.multiplatform.isCommonSource
+import java.io.File
+
+fun CompilerConfiguration.report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation? = null) {
+    get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)?.report(severity, message, location)
+}
+
+fun createSourceFilesFromSourceRoots(
+    configuration: CompilerConfiguration,
+    project: Project,
+    sourceRoots: List<KotlinSourceRoot>,
+    reportLocation: CompilerMessageLocation? = null
+): MutableList<KtFile> {
+    val localFileSystem = VirtualFileManager.getInstance()
+        .getFileSystem(StandardFileSystems.FILE_PROTOCOL)
+    val psiManager = PsiManager.getInstance(project)
+
+    val processedFiles = hashSetOf<VirtualFile>()
+    val result = mutableListOf<KtFile>()
+
+    val virtualFileCreator = PreprocessedFileCreator(project)
+
+    for ((sourceRootPath, isCommon) in sourceRoots) {
+        val vFile = localFileSystem.findFileByPath(sourceRootPath)
+        if (vFile == null) {
+            val message = "Source file or directory not found: $sourceRootPath"
+
+            val buildFilePath = configuration.get(JVMConfigurationKeys.MODULE_XML_FILE)
+            if (buildFilePath != null && Logger.isInitialized()) {
+                Logger.getInstance(KotlinCoreEnvironment::class.java)
+                    .warn("$message\n\nbuild file path: $buildFilePath\ncontent:\n${buildFilePath.readText()}")
+            }
+
+            configuration.report(CompilerMessageSeverity.ERROR, message, reportLocation)
+            continue
+        }
+
+        if (!vFile.isDirectory && vFile.fileType != KotlinFileType.INSTANCE) {
+            configuration.report(CompilerMessageSeverity.ERROR, "Source entry is not a Kotlin file: $sourceRootPath", reportLocation)
+            continue
+        }
+
+        for (file in File(sourceRootPath).walkTopDown()) {
+            if (!file.isFile) continue
+
+            val virtualFile = localFileSystem.findFileByPath(file.absolutePath)?.let(virtualFileCreator::create)
+            if (virtualFile != null && processedFiles.add(virtualFile)) {
+                val psiFile = psiManager.findFile(virtualFile)
+                if (psiFile is KtFile) {
+                    result.add(psiFile)
+                    if (isCommon) {
+                        psiFile.isCommonSource = true
+                    }
+                }
+            }
+        }
+    }
+
+    return result
+}

compiler/frontend/src/org/jetbrains/kotlin/extensions/CollectAdditionalSourcesExtension.kt

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.extensions
+
+import com.intellij.openapi.project.Project
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.psi.KtFile
+
+interface CollectAdditionalSourcesExtension {
+    companion object : ProjectExtensionDescriptor<CollectAdditionalSourcesExtension>(
+        "org.jetbrains.kotlin.collectAdditionalSourcesExtension",
+        CollectAdditionalSourcesExtension::class.java
+    )
+
+    fun collectAdditionalSourcesAndUpdateConfiguration(
+        knownSources: Collection<KtFile>,
+        configuration: CompilerConfiguration,
+        project: Project
+    ): Collection<KtFile>
+}

libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/impl/KJvmCompilerImpl.kt

@@ -23,7 +23,10 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.cli.common.setupCommonArguments
 import org.jetbrains.kotlin.cli.jvm.*
-import org.jetbrains.kotlin.cli.jvm.compiler.*
+import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
+import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
+import org.jetbrains.kotlin.cli.jvm.compiler.NoScopeRecordCliBindingTrace
+import org.jetbrains.kotlin.cli.jvm.compiler.TopDownAnalyzerFacadeForJVM
 import org.jetbrains.kotlin.cli.jvm.config.JvmClasspathRoot
 import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots
 import org.jetbrains.kotlin.cli.jvm.config.jvmClasspathRoots
@@ -41,6 +44,8 @@ import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.psi.KtScript
 import org.jetbrains.kotlin.script.KotlinScriptDefinition
 import org.jetbrains.kotlin.scripting.compiler.plugin.ScriptingCompilerConfigurationComponentRegistrar
+import org.jetbrains.kotlin.scripting.compiler.plugin.dependencies.ScriptsCompilationDependencies
+import org.jetbrains.kotlin.scripting.compiler.plugin.dependencies.collectScriptsCompilationDependencies
 import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull
 import java.util.*
 import kotlin.reflect.KClass
@@ -110,7 +115,11 @@ class KJvmCompilerImpl(val hostConfiguration: ScriptingHostConfiguration) : KJvm
 
             val sourceFiles = arrayListOf(mainKtFile)
             val (classpath, newSources, sourceDependencies) =
-                collectScriptsCompilationDependencies(kotlinCompilerConfiguration, environment.project, sourceFiles)
+                collectScriptsCompilationDependencies(
+                    kotlinCompilerConfiguration,
+                    environment.project,
+                    sourceFiles
+                )
 
             // TODO: consider removing, it is probably redundant: the actual index update is performed with environment.updateClasspath
             kotlinCompilerConfiguration.addJvmClasspathRoots(classpath)

plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/ScriptingCollectAdditionalSourcesExtension.kt

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.scripting.compiler.plugin
+
+import com.intellij.mock.MockProject
+import com.intellij.openapi.project.Project
+import org.jetbrains.kotlin.scripting.compiler.plugin.dependencies.collectScriptsCompilationDependencies
+import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.extensions.CollectAdditionalSourcesExtension
+import org.jetbrains.kotlin.psi.KtFile
+
+class ScriptingCollectAdditionalSourcesExtension(val project: MockProject) : CollectAdditionalSourcesExtension {
+    override fun collectAdditionalSourcesAndUpdateConfiguration(
+        knownSources: Collection<KtFile>,
+        configuration: CompilerConfiguration,
+        project: Project
+    ): Collection<KtFile> {
+        val (newSourcesClasspath, newSources, _) = collectScriptsCompilationDependencies(
+            configuration,
+            project,
+            knownSources
+        )
+        configuration.addJvmClasspathRoots(newSourcesClasspath)
+        return newSources
+    }
+}

plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/dependencies/ScriptsCompilationDependencies.kt

@@ -1,13 +1,15 @@
 /*
- * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
  * that can be found in the license/LICENSE.txt file.
  */
 
-package org.jetbrains.kotlin.cli.jvm.compiler
+package org.jetbrains.kotlin.scripting.compiler.plugin.dependencies
 
 import com.intellij.openapi.project.Project
 import org.jetbrains.kotlin.cli.common.config.KotlinSourceRoot
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
+import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
+import org.jetbrains.kotlin.cli.jvm.compiler.createSourceFilesFromSourceRoots
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.script.ScriptDependenciesProvider
@@ -48,13 +50,18 @@ fun collectScriptsCompilationDependencies(
                         KotlinSourceRoot(it.path, false)
                     }
                     val sourceDependencies =
-                        KotlinCoreEnvironment.createSourceFilesFromSourceRoots(
+                        createSourceFilesFromSourceRoots(
                             configuration, project, sourceDependenciesRoots,
                             // TODO: consider receiving and using precise location from the resolver in the future
                             source.virtualFile?.path?.let { CompilerMessageLocation.create(it) }
                         )
                     if (sourceDependencies.isNotEmpty()) {
-                        collectedSourceDependencies.add(ScriptsCompilationDependencies.SourceDependencies(source, sourceDependencies))
+                        collectedSourceDependencies.add(
+                            ScriptsCompilationDependencies.SourceDependencies(
+                                source,
+                                sourceDependencies
+                            )
+                        )
 
                         val newSources = sourceDependencies.filterNot { knownSourcePaths.contains(it.virtualFile.path) }
                         for (newSource in newSources) {

plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/pluginRegisrar.kt

@@ -9,6 +9,7 @@ import com.intellij.mock.MockProject
 import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
 import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar
 import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.extensions.CollectAdditionalSourcesExtension
 import org.jetbrains.kotlin.extensions.CompilerConfigurationExtension
 import org.jetbrains.kotlin.resolve.extensions.SyntheticResolveExtension
 import org.jetbrains.kotlin.script.ScriptDefinitionProvider
@@ -20,11 +21,8 @@ import org.jetbrains.kotlin.scripting.legacy.CliScriptReportSink
 
 class ScriptingCompilerConfigurationComponentRegistrar : ComponentRegistrar {
     override fun registerProjectComponents(project: MockProject, configuration: CompilerConfiguration) {
-        CompilerConfigurationExtension.registerExtension(project,
+        CompilerConfigurationExtension.registerExtension(project, ScriptingCompilerConfigurationExtension(project))
-                                                         ScriptingCompilerConfigurationExtension(
+        CollectAdditionalSourcesExtension.registerExtension(project, ScriptingCollectAdditionalSourcesExtension(project))
-                                                                                             project
-                                                                                         )
-        )
 
         val scriptDefinitionProvider = CliScriptDefinitionProvider()
         project.registerService(ScriptDefinitionProvider::class.java, scriptDefinitionProvider)
@@ -32,9 +30,7 @@ class ScriptingCompilerConfigurationComponentRegistrar : ComponentRegistrar {
             ScriptDependenciesProvider::class.java,
             CliScriptDependenciesProvider(project)
         )
-        SyntheticResolveExtension.registerExtension(project,
+        SyntheticResolveExtension.registerExtension(project, ScriptingResolveExtension())
-                                                    ScriptingResolveExtension()
-        )
 
         val messageCollector = configuration.get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)
         if (messageCollector != null) {