From 196e274929eb2da575ddd027fbeeec5aaa21fcc7 Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Fri, 15 Feb 2019 17:52:49 +0100 Subject: [PATCH] Move calculating dependencies from script sources to plugin --- .../cli/jvm/compiler/KotlinCoreEnvironment.kt | 99 ++++++------------- .../cli/jvm/compiler/coreEnvironmentUtils.kt | 82 +++++++++++++++ .../CollectAdditionalSourcesExtension.kt | 23 +++++ .../jvmhost/impl/KJvmCompilerImpl.kt | 13 ++- ...iptingCollectAdditionalSourcesExtension.kt | 30 ++++++ .../ScriptsCompilationDependencies.kt | 15 ++- .../compiler/plugin/pluginRegisrar.kt | 12 +-- 7 files changed, 191 insertions(+), 83 deletions(-) create mode 100644 compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/coreEnvironmentUtils.kt create mode 100644 compiler/frontend/src/org/jetbrains/kotlin/extensions/CollectAdditionalSourcesExtension.kt create mode 100644 plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/ScriptingCollectAdditionalSourcesExtension.kt rename {compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler => plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/dependencies}/ScriptsCompilationDependencies.kt (83%) diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt index 18e4397fa21..be53b95e09f 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt @@ -74,7 +74,6 @@ import org.jetbrains.kotlin.cli.common.KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PRO import org.jetbrains.kotlin.cli.common.config.ContentRoot import org.jetbrains.kotlin.cli.common.config.KotlinSourceRoot import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots -import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.STRONG_WARNING @@ -112,7 +111,6 @@ import org.jetbrains.kotlin.resolve.jvm.extensions.PackageFragmentProviderExtens import org.jetbrains.kotlin.resolve.jvm.modules.JavaModuleResolver import org.jetbrains.kotlin.resolve.lazy.declarations.CliDeclarationProviderFactoryService import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactoryService -import org.jetbrains.kotlin.resolve.multiplatform.isCommonSource import org.jetbrains.kotlin.utils.PathUtil import java.io.File import java.lang.reflect.Field @@ -199,9 +197,7 @@ class KotlinCoreEnvironment private constructor( sourceFiles += createKtFiles(project) - val (newSourcesClasspath, newSources, _) = collectScriptsCompilationDependencies(configuration, project, sourceFiles) - configuration.addJvmClasspathRoots(newSourcesClasspath) - sourceFiles += newSources + collectAdditionalSources(project) sourceFiles.sortBy { it.virtualFile.path } @@ -267,6 +263,32 @@ class KotlinCoreEnvironment private constructor( }) } + private fun collectAdditionalSources(project: MockProject) { + var unprocessedSources: Collection = sourceFiles + val processedSources = HashSet() + val processedSourcesByExtension = HashMap>() + // repeat feeding extensions with sources while new sources a being added + var sourceCollectionIterations = 0 + while (unprocessedSources.isNotEmpty()) { + if (sourceCollectionIterations++ > 10) { // TODO: consider using some appropriate global constant + throw IllegalStateException("Unable to collect additional sources in reasonable number of iterations") + } + processedSources.addAll(unprocessedSources) + val allNewSources = ArrayList() + for (extension in CollectAdditionalSourcesExtension.getInstances(project)) { + // do not feed the extension with the sources it returned on the previous iteration + val sourcesToProcess = unprocessedSources - (processedSourcesByExtension[extension] ?: emptyList()) + val newSources = extension.collectAdditionalSourcesAndUpdateConfiguration(sourcesToProcess, configuration, project) + if (newSources.isNotEmpty()) { + allNewSources.addAll(newSources) + processedSourcesByExtension[extension] = newSources + } + } + unprocessedSources = allNewSources.filterNot { processedSources.contains(it) }.distinct() + sourceFiles += unprocessedSources + } + } + fun createPackagePartProvider(scope: GlobalSearchScope): JvmPackagePartProvider { return JvmPackagePartProvider(configuration.languageVersionSettings, scope).apply { addRoots(initialRoots, configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)) @@ -383,7 +405,7 @@ class KotlinCoreEnvironment private constructor( private fun createKtFiles(project: Project): List = createSourceFilesFromSourceRoots(configuration, project, getSourceRootsCheckingForDuplicates()) - internal fun report(severity: CompilerMessageSeverity, message: String) = report(configuration, severity, message) + internal fun report(severity: CompilerMessageSeverity, message: String) = configuration.report(severity, message) companion object { private val LOG = Logger.getInstance(KotlinCoreEnvironment::class.java) @@ -436,67 +458,6 @@ class KotlinCoreEnvironment private constructor( // used in the daemon for jar cache cleanup val applicationEnvironment: JavaCoreApplicationEnvironment? get() = ourApplicationEnvironment - internal fun report( - configuration: CompilerConfiguration, - severity: CompilerMessageSeverity, - message: String, - location: CompilerMessageLocation? = null - ) { - configuration.getNotNull(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY).report(severity, message, location) - } - - internal fun createSourceFilesFromSourceRoots( - configuration: CompilerConfiguration, - project: Project, - sourceRoots: List, - reportLocation: CompilerMessageLocation? = null - ): MutableList { - val localFileSystem = VirtualFileManager.getInstance().getFileSystem(StandardFileSystems.FILE_PROTOCOL) - val psiManager = PsiManager.getInstance(project) - - val processedFiles = hashSetOf() - val result = mutableListOf() - - val virtualFileCreator = PreprocessedFileCreator(project) - - for ((sourceRootPath, isCommon) in sourceRoots) { - val vFile = localFileSystem.findFileByPath(sourceRootPath) - if (vFile == null) { - val message = "Source file or directory not found: $sourceRootPath" - - val buildFilePath = configuration.get(JVMConfigurationKeys.MODULE_XML_FILE) - if (buildFilePath != null && Logger.isInitialized()) { - KotlinCoreEnvironment.LOG.warn("$message\n\nbuild file path: $buildFilePath\ncontent:\n${buildFilePath.readText()}") - } - - report(configuration, ERROR, message, reportLocation) - continue - } - - if (!vFile.isDirectory && vFile.fileType != KotlinFileType.INSTANCE) { - report(configuration, ERROR, "Source entry is not a Kotlin file: $sourceRootPath", reportLocation) - continue - } - - for (file in File(sourceRootPath).walkTopDown()) { - if (!file.isFile) continue - - val virtualFile = localFileSystem.findFileByPath(file.absolutePath)?.let(virtualFileCreator::create) - if (virtualFile != null && processedFiles.add(virtualFile)) { - val psiFile = psiManager.findFile(virtualFile) - if (psiFile is KtFile) { - result.add(psiFile) - if (isCommon) { - psiFile.isCommonSource = true - } - } - } - } - } - - return result - } - private fun getOrCreateApplicationEnvironmentForProduction(configuration: CompilerConfiguration): JavaCoreApplicationEnvironment { synchronized(APPLICATION_LOCK) { if (ourApplicationEnvironment != null) @@ -632,6 +593,7 @@ class KotlinCoreEnvironment private constructor( PreprocessedVirtualFileFactoryExtension.registerExtensionPoint(project) JsSyntheticTranslateExtension.registerExtensionPoint(project) CompilerConfigurationExtension.registerExtensionPoint(project) + CollectAdditionalSourcesExtension.registerExtensionPoint(project) IrGenerationExtension.registerExtensionPoint(project) } @@ -728,8 +690,7 @@ class KotlinCoreEnvironment private constructor( if (!CoreJrtFileSystem.isModularJdk(javaRoot)) { if (classesRoots.isEmpty()) { - val messageCollector = get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY) - messageCollector?.report(ERROR, "No class roots are found in the JDK path: $javaRoot") + report(ERROR, "No class roots are found in the JDK path: $javaRoot") } else { addJvmSdkRoots(classesRoots) } diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/coreEnvironmentUtils.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/coreEnvironmentUtils.kt new file mode 100644 index 00000000000..06992a832f2 --- /dev/null +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/coreEnvironmentUtils.kt @@ -0,0 +1,82 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.cli.jvm.compiler + +import com.intellij.openapi.diagnostic.Logger +import com.intellij.openapi.project.Project +import com.intellij.openapi.vfs.StandardFileSystems +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.openapi.vfs.VirtualFileManager +import com.intellij.psi.PsiManager +import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys +import org.jetbrains.kotlin.cli.common.config.KotlinSourceRoot +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.config.JVMConfigurationKeys +import org.jetbrains.kotlin.extensions.PreprocessedFileCreator +import org.jetbrains.kotlin.idea.KotlinFileType +import org.jetbrains.kotlin.psi.KtFile +import org.jetbrains.kotlin.resolve.multiplatform.isCommonSource +import java.io.File + +fun CompilerConfiguration.report(severity: CompilerMessageSeverity, message: String, location: CompilerMessageLocation? = null) { + get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)?.report(severity, message, location) +} + +fun createSourceFilesFromSourceRoots( + configuration: CompilerConfiguration, + project: Project, + sourceRoots: List, + reportLocation: CompilerMessageLocation? = null +): MutableList { + val localFileSystem = VirtualFileManager.getInstance() + .getFileSystem(StandardFileSystems.FILE_PROTOCOL) + val psiManager = PsiManager.getInstance(project) + + val processedFiles = hashSetOf() + val result = mutableListOf() + + val virtualFileCreator = PreprocessedFileCreator(project) + + for ((sourceRootPath, isCommon) in sourceRoots) { + val vFile = localFileSystem.findFileByPath(sourceRootPath) + if (vFile == null) { + val message = "Source file or directory not found: $sourceRootPath" + + val buildFilePath = configuration.get(JVMConfigurationKeys.MODULE_XML_FILE) + if (buildFilePath != null && Logger.isInitialized()) { + Logger.getInstance(KotlinCoreEnvironment::class.java) + .warn("$message\n\nbuild file path: $buildFilePath\ncontent:\n${buildFilePath.readText()}") + } + + configuration.report(CompilerMessageSeverity.ERROR, message, reportLocation) + continue + } + + if (!vFile.isDirectory && vFile.fileType != KotlinFileType.INSTANCE) { + configuration.report(CompilerMessageSeverity.ERROR, "Source entry is not a Kotlin file: $sourceRootPath", reportLocation) + continue + } + + for (file in File(sourceRootPath).walkTopDown()) { + if (!file.isFile) continue + + val virtualFile = localFileSystem.findFileByPath(file.absolutePath)?.let(virtualFileCreator::create) + if (virtualFile != null && processedFiles.add(virtualFile)) { + val psiFile = psiManager.findFile(virtualFile) + if (psiFile is KtFile) { + result.add(psiFile) + if (isCommon) { + psiFile.isCommonSource = true + } + } + } + } + } + + return result +} \ No newline at end of file diff --git a/compiler/frontend/src/org/jetbrains/kotlin/extensions/CollectAdditionalSourcesExtension.kt b/compiler/frontend/src/org/jetbrains/kotlin/extensions/CollectAdditionalSourcesExtension.kt new file mode 100644 index 00000000000..803d19752a9 --- /dev/null +++ b/compiler/frontend/src/org/jetbrains/kotlin/extensions/CollectAdditionalSourcesExtension.kt @@ -0,0 +1,23 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.extensions + +import com.intellij.openapi.project.Project +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.psi.KtFile + +interface CollectAdditionalSourcesExtension { + companion object : ProjectExtensionDescriptor( + "org.jetbrains.kotlin.collectAdditionalSourcesExtension", + CollectAdditionalSourcesExtension::class.java + ) + + fun collectAdditionalSourcesAndUpdateConfiguration( + knownSources: Collection, + configuration: CompilerConfiguration, + project: Project + ): Collection +} diff --git a/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/impl/KJvmCompilerImpl.kt b/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/impl/KJvmCompilerImpl.kt index 5b8658e0d19..99b82ae3c99 100644 --- a/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/impl/KJvmCompilerImpl.kt +++ b/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/impl/KJvmCompilerImpl.kt @@ -23,7 +23,10 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity import org.jetbrains.kotlin.cli.common.messages.MessageCollector import org.jetbrains.kotlin.cli.common.setupCommonArguments import org.jetbrains.kotlin.cli.jvm.* -import org.jetbrains.kotlin.cli.jvm.compiler.* +import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles +import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment +import org.jetbrains.kotlin.cli.jvm.compiler.NoScopeRecordCliBindingTrace +import org.jetbrains.kotlin.cli.jvm.compiler.TopDownAnalyzerFacadeForJVM import org.jetbrains.kotlin.cli.jvm.config.JvmClasspathRoot import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots import org.jetbrains.kotlin.cli.jvm.config.jvmClasspathRoots @@ -41,6 +44,8 @@ import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.psi.KtScript import org.jetbrains.kotlin.script.KotlinScriptDefinition import org.jetbrains.kotlin.scripting.compiler.plugin.ScriptingCompilerConfigurationComponentRegistrar +import org.jetbrains.kotlin.scripting.compiler.plugin.dependencies.ScriptsCompilationDependencies +import org.jetbrains.kotlin.scripting.compiler.plugin.dependencies.collectScriptsCompilationDependencies import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull import java.util.* import kotlin.reflect.KClass @@ -110,7 +115,11 @@ class KJvmCompilerImpl(val hostConfiguration: ScriptingHostConfiguration) : KJvm val sourceFiles = arrayListOf(mainKtFile) val (classpath, newSources, sourceDependencies) = - collectScriptsCompilationDependencies(kotlinCompilerConfiguration, environment.project, sourceFiles) + collectScriptsCompilationDependencies( + kotlinCompilerConfiguration, + environment.project, + sourceFiles + ) // TODO: consider removing, it is probably redundant: the actual index update is performed with environment.updateClasspath kotlinCompilerConfiguration.addJvmClasspathRoots(classpath) diff --git a/plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/ScriptingCollectAdditionalSourcesExtension.kt b/plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/ScriptingCollectAdditionalSourcesExtension.kt new file mode 100644 index 00000000000..3e6ef3bbf0f --- /dev/null +++ b/plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/ScriptingCollectAdditionalSourcesExtension.kt @@ -0,0 +1,30 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.scripting.compiler.plugin + +import com.intellij.mock.MockProject +import com.intellij.openapi.project.Project +import org.jetbrains.kotlin.scripting.compiler.plugin.dependencies.collectScriptsCompilationDependencies +import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots +import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.extensions.CollectAdditionalSourcesExtension +import org.jetbrains.kotlin.psi.KtFile + +class ScriptingCollectAdditionalSourcesExtension(val project: MockProject) : CollectAdditionalSourcesExtension { + override fun collectAdditionalSourcesAndUpdateConfiguration( + knownSources: Collection, + configuration: CompilerConfiguration, + project: Project + ): Collection { + val (newSourcesClasspath, newSources, _) = collectScriptsCompilationDependencies( + configuration, + project, + knownSources + ) + configuration.addJvmClasspathRoots(newSourcesClasspath) + return newSources + } +} diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/ScriptsCompilationDependencies.kt b/plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/dependencies/ScriptsCompilationDependencies.kt similarity index 83% rename from compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/ScriptsCompilationDependencies.kt rename to plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/dependencies/ScriptsCompilationDependencies.kt index 1e39e20cf45..4bd209ac61b 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/ScriptsCompilationDependencies.kt +++ b/plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/dependencies/ScriptsCompilationDependencies.kt @@ -1,13 +1,15 @@ /* - * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license * that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.cli.jvm.compiler +package org.jetbrains.kotlin.scripting.compiler.plugin.dependencies import com.intellij.openapi.project.Project import org.jetbrains.kotlin.cli.common.config.KotlinSourceRoot import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation +import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment +import org.jetbrains.kotlin.cli.jvm.compiler.createSourceFilesFromSourceRoots import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.script.ScriptDependenciesProvider @@ -48,13 +50,18 @@ fun collectScriptsCompilationDependencies( KotlinSourceRoot(it.path, false) } val sourceDependencies = - KotlinCoreEnvironment.createSourceFilesFromSourceRoots( + createSourceFilesFromSourceRoots( configuration, project, sourceDependenciesRoots, // TODO: consider receiving and using precise location from the resolver in the future source.virtualFile?.path?.let { CompilerMessageLocation.create(it) } ) if (sourceDependencies.isNotEmpty()) { - collectedSourceDependencies.add(ScriptsCompilationDependencies.SourceDependencies(source, sourceDependencies)) + collectedSourceDependencies.add( + ScriptsCompilationDependencies.SourceDependencies( + source, + sourceDependencies + ) + ) val newSources = sourceDependencies.filterNot { knownSourcePaths.contains(it.virtualFile.path) } for (newSource in newSources) { diff --git a/plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/pluginRegisrar.kt b/plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/pluginRegisrar.kt index 66cca47a5ef..2e03021bc9e 100644 --- a/plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/pluginRegisrar.kt +++ b/plugins/scripting/scripting-compiler/src/org/jetbrains/kotlin/scripting/compiler/plugin/pluginRegisrar.kt @@ -9,6 +9,7 @@ import com.intellij.mock.MockProject import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar import org.jetbrains.kotlin.config.CompilerConfiguration +import org.jetbrains.kotlin.extensions.CollectAdditionalSourcesExtension import org.jetbrains.kotlin.extensions.CompilerConfigurationExtension import org.jetbrains.kotlin.resolve.extensions.SyntheticResolveExtension import org.jetbrains.kotlin.script.ScriptDefinitionProvider @@ -20,11 +21,8 @@ import org.jetbrains.kotlin.scripting.legacy.CliScriptReportSink class ScriptingCompilerConfigurationComponentRegistrar : ComponentRegistrar { override fun registerProjectComponents(project: MockProject, configuration: CompilerConfiguration) { - CompilerConfigurationExtension.registerExtension(project, - ScriptingCompilerConfigurationExtension( - project - ) - ) + CompilerConfigurationExtension.registerExtension(project, ScriptingCompilerConfigurationExtension(project)) + CollectAdditionalSourcesExtension.registerExtension(project, ScriptingCollectAdditionalSourcesExtension(project)) val scriptDefinitionProvider = CliScriptDefinitionProvider() project.registerService(ScriptDefinitionProvider::class.java, scriptDefinitionProvider) @@ -32,9 +30,7 @@ class ScriptingCompilerConfigurationComponentRegistrar : ComponentRegistrar { ScriptDependenciesProvider::class.java, CliScriptDependenciesProvider(project) ) - SyntheticResolveExtension.registerExtension(project, - ScriptingResolveExtension() - ) + SyntheticResolveExtension.registerExtension(project, ScriptingResolveExtension()) val messageCollector = configuration.get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY) if (messageCollector != null) {