From 942c8aaafc6a2c6acecf201b6f52b186d3427ba5 Mon Sep 17 00:00:00 2001 From: Natalia Selezneva Date: Wed, 12 Feb 2020 14:19:20 +0300 Subject: [PATCH] Introduce index for folders with script definitions templates Replace it with nonBlockingReadAction meaning that if update was canceled then loading will be restarted after all write events happen Also scanning is now started after indexing ^KT-36378 Fixed ^KT-34138 Fixed ^KT-37863 Fixed --- idea/resources/META-INF/scripting-support.xml | 1 + .../AsyncScriptDefinitionsContributor.kt | 113 ---------- .../script/ScriptTemplatesClassRootsIndex.kt | 65 ++++++ ...ScriptTemplatesFromDependenciesProvider.kt | 196 ++++++++++++++++++ .../scriptsTemplatesFromDependencies.kt | 131 ------------ 5 files changed, 262 insertions(+), 244 deletions(-) delete mode 100644 idea/src/org/jetbrains/kotlin/idea/script/AsyncScriptDefinitionsContributor.kt create mode 100644 idea/src/org/jetbrains/kotlin/idea/script/ScriptTemplatesClassRootsIndex.kt create mode 100644 idea/src/org/jetbrains/kotlin/idea/script/ScriptTemplatesFromDependenciesProvider.kt delete mode 100644 idea/src/org/jetbrains/kotlin/idea/script/scriptsTemplatesFromDependencies.kt diff --git a/idea/resources/META-INF/scripting-support.xml b/idea/resources/META-INF/scripting-support.xml index 6b84786709f..e2c1e454ba1 100644 --- a/idea/resources/META-INF/scripting-support.xml +++ b/idea/resources/META-INF/scripting-support.xml @@ -9,5 +9,6 @@ + \ No newline at end of file diff --git a/idea/src/org/jetbrains/kotlin/idea/script/AsyncScriptDefinitionsContributor.kt b/idea/src/org/jetbrains/kotlin/idea/script/AsyncScriptDefinitionsContributor.kt deleted file mode 100644 index 5629e31c502..00000000000 --- a/idea/src/org/jetbrains/kotlin/idea/script/AsyncScriptDefinitionsContributor.kt +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.idea.script - -import com.intellij.openapi.progress.ProgressIndicator -import com.intellij.openapi.progress.Task -import com.intellij.openapi.project.Project -import org.jetbrains.kotlin.idea.core.script.ScriptDefinitionSourceAsContributor -import org.jetbrains.kotlin.idea.core.script.ScriptDefinitionsManager -import org.jetbrains.kotlin.idea.util.application.executeOnPooledThread -import org.jetbrains.kotlin.scripting.definitions.ScriptDefinition -import java.util.concurrent.locks.ReentrantReadWriteLock -import kotlin.concurrent.read -import kotlin.concurrent.write - -abstract class AsyncScriptDefinitionsContributor(protected val project: Project) : ScriptDefinitionSourceAsContributor { - abstract val progressMessage: String - - override fun isReady(): Boolean = _definitions != null - - override val definitions: Sequence - get() { - definitionsLock.read { - if (_definitions != null) { - return _definitions!!.asSequence() - } - } - - forceStartUpdate = false - asyncRunUpdateScriptTemplates() - return emptySequence() - } - - protected fun asyncRunUpdateScriptTemplates() { - val backgroundTask = inProgressLock.write { - shouldStartNewUpdate = true - if (!inProgress) { - inProgress = true - return@write DefinitionsCollectorBackgroundTask() - } - return@write null - } - // TODO: resolve actual reason for the exception below - try { - backgroundTask?.queue() - } catch (e: IllegalStateException) { - if (e.message?.contains("Calling invokeAndWait from read-action leads to possible deadlock") == false) throw e - } - } - - protected abstract fun loadScriptDefinitions(previous: List?): List - - private var _definitions: List? = null - private val definitionsLock = ReentrantReadWriteLock() - - @Volatile - protected var forceStartUpdate = false - - @Volatile - protected var shouldStartNewUpdate = false - - private var inProgress = false - private val inProgressLock = ReentrantReadWriteLock() - - private inner class DefinitionsCollectorBackgroundTask : Task.Backgroundable(project, progressMessage, true) { - override fun onFinished() { - inProgressLock.write { - inProgress = false - } - } - - override fun run(indicator: ProgressIndicator) { - while (true) { - if (indicator.isCanceled || !shouldStartNewUpdate) { - return - } - - shouldStartNewUpdate = false - - val previousDefinitions = definitionsLock.read { - if (!forceStartUpdate && _definitions != null) return - - _definitions - } - - val newDefinitions = loadScriptDefinitions(previousDefinitions) - - val needReload = definitionsLock.write { - if (newDefinitions != _definitions) { - _definitions = newDefinitions - return@write true - } - return@write false - } - - if (needReload) { - if (isHeadless) { - // If new script definitions found, then ScriptDefinitionsManager.reloadDefinitionsBy should be called - // This may cause deadlock because Task.Backgroundable.queue executes task synchronously in headless mode - executeOnPooledThread { - ScriptDefinitionsManager.getInstance(project).reloadDefinitionsBy(this@AsyncScriptDefinitionsContributor) - } - } else { - ScriptDefinitionsManager.getInstance(project).reloadDefinitionsBy(this@AsyncScriptDefinitionsContributor) - } - } - } - } - } -} \ No newline at end of file diff --git a/idea/src/org/jetbrains/kotlin/idea/script/ScriptTemplatesClassRootsIndex.kt b/idea/src/org/jetbrains/kotlin/idea/script/ScriptTemplatesClassRootsIndex.kt new file mode 100644 index 00000000000..67b84a664f9 --- /dev/null +++ b/idea/src/org/jetbrains/kotlin/idea/script/ScriptTemplatesClassRootsIndex.kt @@ -0,0 +1,65 @@ +/* + * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.idea.script + +import com.intellij.openapi.vfs.VirtualFile +import com.intellij.util.indexing.* +import com.intellij.util.io.IOUtil +import com.intellij.util.io.KeyDescriptor +import org.jetbrains.kotlin.scripting.definitions.SCRIPT_DEFINITION_MARKERS_PATH +import java.io.DataInput +import java.io.DataOutput +import java.util.* + +class ScriptTemplatesClassRootsIndex : + ScalarIndexExtension(), + FileBasedIndex.InputFilter, KeyDescriptor, + DataIndexer { + + companion object { + val KEY = ID.create(ScriptTemplatesClassRootsIndex::class.java.canonicalName) + + private val suffix = SCRIPT_DEFINITION_MARKERS_PATH.removeSuffix("/") + } + + override fun getName(): ID = KEY + + override fun getIndexer(): DataIndexer = this + + override fun getKeyDescriptor(): KeyDescriptor = this + + override fun getInputFilter(): FileBasedIndex.InputFilter = this + + override fun dependsOnFileContent() = false + + override fun getVersion(): Int = 1 + + override fun indexDirectories(): Boolean = true + + override fun acceptInput(file: VirtualFile): Boolean { + return file.isDirectory && file.path.endsWith(suffix) + } + + override fun save(out: DataOutput, value: String) { + IOUtil.writeUTF(out, value) + } + + override fun read(input: DataInput): String? { + return IOUtil.readUTF(input) + } + + override fun getHashCode(value: String): Int { + return value.hashCode() + } + + override fun isEqual(val1: String?, val2: String?): Boolean { + return val1 == val2 + } + + override fun map(inputData: FileContent): Map { + return Collections.singletonMap(inputData.file.url, null) + } +} \ No newline at end of file diff --git a/idea/src/org/jetbrains/kotlin/idea/script/ScriptTemplatesFromDependenciesProvider.kt b/idea/src/org/jetbrains/kotlin/idea/script/ScriptTemplatesFromDependenciesProvider.kt new file mode 100644 index 00000000000..28686c2a9f4 --- /dev/null +++ b/idea/src/org/jetbrains/kotlin/idea/script/ScriptTemplatesFromDependenciesProvider.kt @@ -0,0 +1,196 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.idea.script + +import com.intellij.ProjectTopics +import com.intellij.openapi.application.ApplicationManager +import com.intellij.openapi.application.ReadAction +import com.intellij.openapi.project.Project +import com.intellij.openapi.roots.ModuleRootEvent +import com.intellij.openapi.roots.ModuleRootListener +import com.intellij.openapi.roots.OrderEnumerator +import com.intellij.openapi.roots.ProjectFileIndex +import com.intellij.openapi.vfs.* +import com.intellij.util.concurrency.AppExecutorUtil +import com.intellij.util.indexing.FileBasedIndex +import org.jetbrains.kotlin.idea.core.script.ScriptDefinitionSourceAsContributor +import org.jetbrains.kotlin.idea.core.script.ScriptDefinitionsManager +import org.jetbrains.kotlin.idea.core.script.loadDefinitionsFromTemplates +import org.jetbrains.kotlin.scripting.definitions.SCRIPT_DEFINITION_MARKERS_EXTENSION_WITH_DOT +import org.jetbrains.kotlin.scripting.definitions.SCRIPT_DEFINITION_MARKERS_PATH +import org.jetbrains.kotlin.scripting.definitions.ScriptDefinition +import org.jetbrains.kotlin.scripting.definitions.getEnvironment +import java.io.File +import java.net.URL +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock +import kotlin.script.experimental.host.ScriptingHostConfiguration +import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration + + +class ScriptTemplatesFromDependenciesProvider(private val project: Project) : ScriptDefinitionSourceAsContributor { + override val id = "ScriptTemplatesFromDependenciesProvider" + + override fun isReady(): Boolean = _definitions != null + + override val definitions: Sequence + get() { + definitionsLock.withLock { + if (_definitions != null) { + return _definitions!!.asSequence() + } + } + + forceStartUpdate = false + asyncRunUpdateScriptTemplates() + return emptySequence() + } + + init { + val connection = project.messageBus.connect() + connection.subscribe( + ProjectTopics.PROJECT_ROOTS, + object : ModuleRootListener { + override fun rootsChanged(event: ModuleRootEvent) { + if (project.isInitialized) { + forceStartUpdate = true + asyncRunUpdateScriptTemplates() + } + } + }, + ) + } + + private fun asyncRunUpdateScriptTemplates() { + definitionsLock.withLock { + if (!forceStartUpdate && _definitions != null) return + } + + inProgressLock.withLock { + if (!inProgress) { + inProgress = true + + loadScriptDefinitions() + } + } + } + + private var _definitions: List? = null + private val definitionsLock = ReentrantLock() + + private var oldTemplates: TemplatesWithCp? = null + + private data class TemplatesWithCp( + val templates: List, + val classpath: List, + ) + + private var inProgress = false + private val inProgressLock = ReentrantLock() + + @Volatile + private var forceStartUpdate = false + + private fun loadScriptDefinitions() { + if (ApplicationManager.getApplication().isUnitTestMode || project.isDefault) { + return onEarlyEnd() + } + + val templates = LinkedHashSet() + val classpath = LinkedHashSet() + + ReadAction + .nonBlocking> { + val fileManager = VirtualFileManager.getInstance() + FileBasedIndex.getInstance().getAllKeys(ScriptTemplatesClassRootsIndex.KEY, project).mapNotNull { + val vFile = fileManager.findFileByUrl(it) + + // see SCRIPT_DEFINITION_MARKERS_PATH + vFile?.parent?.parent?.parent?.parent + } + } + .inSmartMode(project) + .expireWith(project) + .submit(AppExecutorUtil.getAppExecutorService()) + .onSuccess { roots -> + val jarFS = JarFileSystem.getInstance() + roots.forEach { root -> + root.findFileByRelativePath(SCRIPT_DEFINITION_MARKERS_PATH)?.children?.forEach { resourceFile -> + if (resourceFile.isValid && !resourceFile.isDirectory) { + templates.add(resourceFile.name.removeSuffix(SCRIPT_DEFINITION_MARKERS_EXTENSION_WITH_DOT)) + } + } + + val templateSource = jarFS.getVirtualFileForJar(root) ?: root + val module = ProjectFileIndex.getInstance(project).getModuleForFile(templateSource) ?: return@forEach + + // assuming that all libraries are placed into classes roots + // TODO: extract exact library dependencies instead of putting all module dependencies into classpath + // minimizing the classpath needed to use the template by taking cp only from modules with new templates found + // on the other hand the approach may fail if some module contains a template without proper classpath, while + // the other has properly configured classpath, so assuming that the dependencies are set correctly everywhere + classpath.addAll( + OrderEnumerator.orderEntries(module).withoutSdk().classesRoots.mapNotNull { + it.canonicalPath?.removeSuffix("!/").let(::File) + } + ) + } + } + .onProcessed { + if (templates.isEmpty()) return@onProcessed onEarlyEnd() + + val newTemplates = TemplatesWithCp(templates.toList(), classpath.toList()) + if (newTemplates == oldTemplates) { + inProgressLock.withLock { + inProgress = false + } + + return@onProcessed + } + + oldTemplates = newTemplates + + val hostConfiguration = ScriptingHostConfiguration(defaultJvmScriptingHostConfiguration) { + getEnvironment { + mapOf( + "projectRoot" to (project.basePath ?: project.baseDir.canonicalPath)?.let(::File), + ) + } + } + + val newDefinitions = loadDefinitionsFromTemplates( + templateClassNames = newTemplates.templates, + templateClasspath = newTemplates.classpath, + baseHostConfiguration = hostConfiguration, + ) + + val needReload = definitionsLock.withLock { + if (newDefinitions != _definitions) { + _definitions = newDefinitions + return@withLock true + } + return@withLock false + } + + if (needReload) { + ScriptDefinitionsManager.getInstance(project).reloadDefinitionsBy(this@ScriptTemplatesFromDependenciesProvider) + } + + inProgressLock.withLock { + inProgress = false + } + } + } + + private fun onEarlyEnd() { + definitionsLock.withLock { + _definitions = emptyList() + } + inProgressLock.withLock { + inProgress = false + } + } +} \ No newline at end of file diff --git a/idea/src/org/jetbrains/kotlin/idea/script/scriptsTemplatesFromDependencies.kt b/idea/src/org/jetbrains/kotlin/idea/script/scriptsTemplatesFromDependencies.kt deleted file mode 100644 index de529d346c2..00000000000 --- a/idea/src/org/jetbrains/kotlin/idea/script/scriptsTemplatesFromDependencies.kt +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2000-2018 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.idea.script - -import com.intellij.ProjectTopics -import com.intellij.openapi.application.runReadAction -import com.intellij.openapi.project.Project -import com.intellij.openapi.roots.ModuleRootEvent -import com.intellij.openapi.roots.ModuleRootListener -import com.intellij.openapi.roots.OrderEnumerator -import com.intellij.openapi.vfs.JarFileSystem -import com.intellij.openapi.vfs.VirtualFile -import org.jetbrains.kotlin.idea.KotlinBundle -import org.jetbrains.kotlin.idea.core.script.loadDefinitionsFromTemplates -import org.jetbrains.kotlin.idea.util.projectStructure.allModules -import org.jetbrains.kotlin.scripting.definitions.SCRIPT_DEFINITION_MARKERS_EXTENSION_WITH_DOT -import org.jetbrains.kotlin.scripting.definitions.SCRIPT_DEFINITION_MARKERS_PATH -import org.jetbrains.kotlin.scripting.definitions.ScriptDefinition -import org.jetbrains.kotlin.scripting.definitions.getEnvironment -import java.io.File -import java.util.concurrent.locks.ReentrantReadWriteLock -import kotlin.concurrent.write -import kotlin.script.experimental.host.ScriptingHostConfiguration -import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration - -class ScriptTemplatesFromDependenciesProvider(project: Project) : AsyncScriptDefinitionsContributor(project) { - - private var templates: TemplatesWithCp? = null - private val templatesLock = ReentrantReadWriteLock() - - init { - val connection = project.messageBus.connect() - connection.subscribe(ProjectTopics.PROJECT_ROOTS, object : ModuleRootListener { - override fun rootsChanged(event: ModuleRootEvent) { - if (project.isInitialized) { - forceStartUpdate = true - asyncRunUpdateScriptTemplates() - } - } - }) - } - - override val id = "ScriptTemplatesFromDependenciesProvider" - override val progressMessage = KotlinBundle.message("script.progress.text.kotlin.scanning.dependencies.for.script.definitions") - - override fun loadScriptDefinitions(previous: List?): List { - if (project.isDefault) return emptyList() - - val templatesCopy = templatesLock.write { - val newTemplates = scriptDefinitionsFromDependencies(project) - if (newTemplates != templates) { - templates = newTemplates - return@write newTemplates - } - return@write null - } - if (templatesCopy != null) { - val hostConfiguration = ScriptingHostConfiguration(defaultJvmScriptingHostConfiguration) { - getEnvironment { - mapOf( - "projectRoot" to (project.basePath ?: project.baseDir.canonicalPath)?.let(::File) - ) - } - } - return loadDefinitionsFromTemplates( - templateClassNames = templatesCopy.templates, - templateClasspath = templatesCopy.classpath, - baseHostConfiguration = hostConfiguration - ) - } - return previous ?: emptyList() - } -} - -private data class TemplatesWithCp( - val templates: List, - val classpath: List -) - -private fun scriptDefinitionsFromDependencies(project: Project): TemplatesWithCp { - val templates = LinkedHashSet() - val classpath = LinkedHashSet() - - fun addTemplatesFromRoot(vfile: VirtualFile): Boolean { - var templatesFound = false - val root = JarFileSystem.getInstance().getJarRootForLocalFile(vfile) ?: vfile - if (root.isValid) { - root.findFileByRelativePath(SCRIPT_DEFINITION_MARKERS_PATH)?.takeIf { it.isDirectory }?.children?.forEach { - if (it.isValid && !it.isDirectory) { - templates.add(it.name.removeSuffix(SCRIPT_DEFINITION_MARKERS_EXTENSION_WITH_DOT)) - templatesFound = true - } - } - } - return templatesFound - } - - runReadAction { - if (project.isDisposed) return@runReadAction - // processing source roots from the same project first since the resources are copied to the classes roots only on compilation - project.allModules().forEach { module -> - OrderEnumerator.orderEntries(module).withoutDepModules().withoutLibraries().withoutSdk().sourceRoots.forEach { root -> - if (addTemplatesFromRoot(root)) { - classpath.addAll(OrderEnumerator.orderEntries(module).withoutSdk().classesRoots.mapNotNull { - it.canonicalPath?.removeSuffix("!/").let(::File) - }) - } - } - } - - project.allModules().forEach { module -> - // assuming that all libraries are placed into classes roots - // TODO: extract exact library dependencies instead of putting all module dependencies into classpath - OrderEnumerator.orderEntries(module).withoutDepModules().withoutSdk().classesRoots.forEach { root -> - if (addTemplatesFromRoot(root)) { - // minimizing the classpath needed to use the template by taking cp only from modules with new templates found - // on the other hand the approach may fail if some module contains a template without proper classpath, while - // the other has properly configured classpath, so assuming that the dependencies are set correctly everywhere - classpath.addAll(OrderEnumerator.orderEntries(module).withoutSdk().classesRoots.mapNotNull { - it.canonicalPath?.removeSuffix("!/").let(::File) - }) - } - } - } - } - return TemplatesWithCp(templates.toList(), classpath.toList()) -} -