Refactor script definitions loading

Make ScriptDefinitionProvider an interface
    and provide different implementation for cli and IDEA
Rework ide extension point to a simpler interface (ScriptDefinitionContributor)
Move template loading logic into a top level function
Allow script definitions to be reloaded in IDE

idea/idea-core/src/org/jetbrains/kotlin/idea/core/script/ScriptDefinitionsManager.kt

@@ -0,0 +1,190 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.idea.core.script
+
+import com.intellij.openapi.components.service
+import com.intellij.openapi.diagnostic.Logger
+import com.intellij.openapi.extensions.ExtensionPointName
+import com.intellij.openapi.extensions.Extensions
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.projectRoots.JavaSdk
+import com.intellij.openapi.projectRoots.ProjectJdkTable
+import com.intellij.openapi.projectRoots.ex.PathUtilEx
+import org.jetbrains.kotlin.script.KotlinScriptDefinition
+import org.jetbrains.kotlin.script.KotlinScriptDefinitionFromAnnotatedTemplate
+import org.jetbrains.kotlin.script.ScriptDefinitionProvider
+import org.jetbrains.kotlin.utils.PathUtil
+import org.jetbrains.kotlin.utils.addToStdlib.flattenTo
+import java.io.File
+import java.net.URLClassLoader
+import java.util.concurrent.locks.ReentrantReadWriteLock
+import kotlin.concurrent.read
+import kotlin.concurrent.write
+import kotlin.script.dependencies.Environment
+import kotlin.script.dependencies.ScriptContents
+import kotlin.script.experimental.dependencies.DependenciesResolver
+import kotlin.script.experimental.dependencies.ScriptDependencies
+import kotlin.script.experimental.dependencies.asSuccess
+import kotlin.script.templates.standard.ScriptTemplateWithArgs
+
+class ScriptDefinitionsManager(private val project: Project): ScriptDefinitionProvider {
+    private val lock = ReentrantReadWriteLock()
+    private var definitionsByContributor = mutableMapOf<ScriptDefinitionContributor, List<KotlinScriptDefinition>>()
+    private var definitions: List<KotlinScriptDefinition> = emptyList()
+
+    fun reloadDefinitionsBy(contributor: ScriptDefinitionContributor) = lock.write {
+        currentDefinitions() //force definitions to be computed
+
+        if (contributor !in definitionsByContributor) error("Unknown contributor: ${contributor.id}")
+
+        definitionsByContributor[contributor] = contributor.safeGetDefinitions()
+        updateDefinitions()
+    }
+
+    private fun currentDefinitions(): List<KotlinScriptDefinition> {
+        val hasDefinitions = definitions.isNotEmpty()
+        when {
+            hasDefinitions -> return definitions
+            else -> {
+                reloadScriptDefinitions()
+                return definitions
+            }
+        }
+    }
+
+    override fun findScriptDefinition(fileName: String): KotlinScriptDefinition? = lock.read {
+        currentDefinitions().firstOrNull { it.isScript(fileName) }
+    }
+
+    override fun isScript(fileName: String) = lock.read {
+        currentDefinitions().any { it.isScript(fileName) }
+    }
+
+    private fun getContributors(): List<ScriptDefinitionContributor> =
+            Extensions.getArea(project).getExtensionPoint(ScriptDefinitionContributor.EP_NAME).extensions.toList()
+
+    fun reloadScriptDefinitions() = lock.write {
+        definitionsByContributor = getContributors().associateByTo(mutableMapOf(), { it }, { it.safeGetDefinitions() })
+        updateDefinitions()
+    }
+
+    private fun updateDefinitions() {
+        definitions = definitionsByContributor.values.flattenTo(mutableListOf())
+        // TODO: clear by script type/definition
+        project.service<ScriptDependenciesUpdater>().clear()
+    }
+
+    private fun ScriptDefinitionContributor.safeGetDefinitions(): List<KotlinScriptDefinition> {
+        return try {
+            getDefinitions()
+        }
+        catch (t: Throwable) {
+            // TODO: review exception handling
+            // possibly log, see KT-19276
+            emptyList()
+        }
+    }
+
+    companion object {
+        fun getInstance(project: Project): ScriptDefinitionsManager = project.service<ScriptDefinitionProvider>() as ScriptDefinitionsManager
+    }
+}
+
+
+private val LOG = Logger.getInstance("ScriptTemplatesProviders")
+
+fun loadDefinitionsFromTemplates(
+        templateClassNames: List<String>,
+        templateClasspath: List<File>,
+        environment: Environment = emptyMap(),
+        // TODO: need to provide a way to specify this in compiler/repl .. etc
+        /*
+         * Allows to specify additional jars needed for DependenciesResolver (and not script template).
+         * Script template dependencies naturally become (part of) dependencies of the script which is not always desired for resolver dependencies.
+         * i.e. gradle resolver may depend on some jars that 'built.gradle.kts' files should not depend on.
+         */
+        additionalResolverClasspath: List<File> = emptyList()
+): List<KotlinScriptDefinitionFromAnnotatedTemplate> = try {
+    val classpath = templateClasspath + additionalResolverClasspath
+    LOG.info("[kts] loading script definitions $templateClassNames using cp: ${classpath.joinToString(File.pathSeparator)}")
+    val baseLoader = ScriptDefinitionContributor::class.java.classLoader
+    val loader = if (classpath.isEmpty()) baseLoader else URLClassLoader(classpath.map { it.toURI().toURL() }.toTypedArray(), baseLoader)
+
+    templateClassNames.map {
+        KotlinScriptDefinitionFromAnnotatedTemplate(
+                loader.loadClass(it).kotlin,
+                environment,
+                templateClasspath
+        )
+    }
+}
+catch (ex: Throwable) {
+    // TODO: review exception handling
+    emptyList()
+}
+
+interface ScriptDefinitionContributor {
+    val id: String
+
+    fun getDefinitions(): List<KotlinScriptDefinition>
+
+    companion object {
+        val EP_NAME: ExtensionPointName<ScriptDefinitionContributor> =
+                ExtensionPointName.create<ScriptDefinitionContributor>("org.jetbrains.kotlin.scriptDefinitionContributor")
+
+        inline fun <reified T> find(project: Project) =
+                Extensions.getArea(project).getExtensionPoint(ScriptDefinitionContributor.EP_NAME).extensions.filterIsInstance<T>().firstOrNull()
+    }
+
+}
+
+class StandardScriptDefinitionContributor(private val project: Project) : ScriptDefinitionContributor {
+    override fun getDefinitions() = listOf(StandardIdeScriptDefinition(project))
+
+    override val id: String = "StandardKotlinScript"
+}
+
+
+class StandardIdeScriptDefinition(project: Project) : KotlinScriptDefinition(ScriptTemplateWithArgs::class) {
+    override val dependencyResolver = BundledKotlinScriptDependenciesResolver(project)
+}
+
+class BundledKotlinScriptDependenciesResolver(private val project: Project) : DependenciesResolver {
+    override fun resolve(
+            scriptContents: ScriptContents,
+            environment: Environment
+    ): DependenciesResolver.ResolveResult {
+        val javaHome = getScriptSDK(project)
+        val dependencies = ScriptDependencies(
+                javaHome = javaHome?.let(::File),
+                classpath = with(PathUtil.kotlinPathsForIdeaPlugin) {
+                    listOf(
+                            reflectPath,
+                            stdlibPath,
+                            scriptRuntimePath
+                    )
+                }
+        )
+        return dependencies.asSuccess()
+    }
+
+    private fun getScriptSDK(project: Project): String? {
+        val jdk = ProjectJdkTable.getInstance().allJdks.firstOrNull { sdk -> sdk.sdkType is JavaSdk } ?: PathUtilEx.getAnyJdk(project)
+        return jdk?.homePath
+    }
+}
+

idea/idea-core/src/org/jetbrains/kotlin/idea/core/script/ScriptDependenciesCache.kt

@@ -17,7 +17,6 @@
 package org.jetbrains.kotlin.idea.core.script
 
 import com.intellij.codeInsight.daemon.DaemonCodeAnalyzer
-import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.extensions.Extensions
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.vfs.VirtualFile
@@ -27,6 +26,7 @@ import com.intellij.psi.search.NonClasspathDirectoriesScope
 import com.intellij.util.containers.SLRUMap
 import kotlinx.coroutines.experimental.launch
 import org.jetbrains.kotlin.idea.core.util.EDT
+import org.jetbrains.kotlin.utils.addIfNotNull
 import java.io.File
 import java.util.concurrent.locks.ReentrantReadWriteLock
 import kotlin.concurrent.read
@@ -37,11 +37,11 @@ import kotlin.reflect.KProperty0
 import kotlin.reflect.jvm.isAccessible
 import kotlin.script.experimental.dependencies.ScriptDependencies
 
-private const val MAX_SCRIPTS_CACHED = 15
+private const val MAX_SCRIPTS_CACHED = 10
 
 class ScriptDependenciesCache(private val project: Project) {
     private val cacheLock = ReentrantReadWriteLock()
-    private val cache = SLRUMap<VirtualFile, ScriptDependencies>(0, MAX_SCRIPTS_CACHED)
+    private val cache = SLRUMap<VirtualFile, ScriptDependencies>(MAX_SCRIPTS_CACHED, MAX_SCRIPTS_CACHED)
 
     operator fun get(virtualFile: VirtualFile): ScriptDependencies? = cacheLock.read { cache[virtualFile] }
 
@@ -62,7 +62,7 @@ class ScriptDependenciesCache(private val project: Project) {
         NonClasspathDirectoriesScope(allLibrarySources)
     }
 
-    private fun onChange(file: VirtualFile?) {
+    private fun onChange(files: List<VirtualFile>) {
         this::allScriptsClasspath.clearValue()
         this::allScriptsClasspathScope.clearValue()
         this::allLibrarySources.clearValue()
@@ -74,22 +74,18 @@ class ScriptDependenciesCache(private val project: Project) {
                         .single()
 
         kotlinScriptDependenciesClassFinder.clearCache()
-        updateHighlighting(file)
+        updateHighlighting(files)
     }
 
-    private fun updateHighlighting(file: VirtualFile?) {
+    private fun updateHighlighting(files: List<VirtualFile>) {
         ScriptDependenciesModificationTracker.getInstance(project).incModificationCount()
 
         launch(EDT(project)) {
-            if (file != null) {
-                file.let { PsiManager.getInstance(project).findFile(it) }?.let { psiFile ->
+            files.forEach {
+                PsiManager.getInstance(project).findFile(it)?.let { psiFile ->
                     DaemonCodeAnalyzer.getInstance(project).restart(psiFile)
                 }
             }
-            else {
-                assert(ApplicationManager.getApplication().isUnitTestMode)
-                DaemonCodeAnalyzer.getInstance(project).restart()
-            }
         }
     }
 
@@ -99,8 +95,13 @@ class ScriptDependenciesCache(private val project: Project) {
     }
 
     fun clear() {
-        cacheLock.write(cache::clear)
-        onChange(null)
+        val keys = cacheLock.read {
+            val keys = mutableListOf<VirtualFile>()
+            cache.iterateKeys { keys.addIfNotNull(it) }
+            cacheLock.write(cache::clear)
+            keys
+        }
+        onChange(keys)
     }
 
     fun save(virtualFile: VirtualFile, new: ScriptDependencies): Boolean {
@@ -111,7 +112,7 @@ class ScriptDependenciesCache(private val project: Project) {
         }
         val changed = new != old
         if (changed) {
-            onChange(virtualFile)
+            onChange(listOf(virtualFile))
         }
 
         return changed
@@ -122,7 +123,7 @@ class ScriptDependenciesCache(private val project: Project) {
             cache.remove(virtualFile) != null
         }
         if (changed) {
-            onChange(virtualFile)
+            onChange(listOf(virtualFile))
         }
         return changed
     }

idea/idea-core/src/org/jetbrains/kotlin/idea/core/script/ScriptDependenciesManager.kt

@@ -23,9 +23,8 @@ import com.intellij.openapi.vfs.StandardFileSystems
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.util.io.URLUtil
 import org.jetbrains.annotations.TestOnly
-import org.jetbrains.kotlin.script.KotlinScriptDefinitionProvider
+import org.jetbrains.kotlin.script.ScriptDefinitionProvider
 import org.jetbrains.kotlin.script.ScriptDependenciesProvider
-import org.jetbrains.kotlin.script.makeScriptDefsFromTemplatesProviderExtensions
 import java.io.File
 import kotlin.script.experimental.dependencies.ScriptDependencies
 
@@ -41,25 +40,12 @@ class IdeScriptDependenciesProvider(
 }
 
 class ScriptDependenciesManager internal constructor(
-        private val project: Project,
-        private val scriptDefinitionProvider: KotlinScriptDefinitionProvider,
         private val cacheUpdater: ScriptDependenciesUpdater,
         private val cache: ScriptDependenciesCache
 ) {
-    init {
-        reloadScriptDefinitions()
-    }
-
     fun getScriptClasspath(file: VirtualFile): List<VirtualFile> = toVfsRoots(cacheUpdater.getCurrentDependencies(file).classpath)
     fun getScriptDependencies(file: VirtualFile): ScriptDependencies = cacheUpdater.getCurrentDependencies(file)
 
-    private fun reloadScriptDefinitions() {
-        val def = makeScriptDefsFromTemplatesProviderExtensions(
-                project, { ep, ex -> /* do nothing, logging these exception creates too much noise*/}
-        )
-        scriptDefinitionProvider.setScriptDefinitions(def)
-    }
-
     fun getAllScriptsClasspathScope() = cache.allScriptsClasspathScope
     fun getAllLibrarySourcesScope() = cache.allLibrarySourcesScope
     fun getAllLibrarySources() = cache.allLibrarySources
@@ -90,18 +76,10 @@ class ScriptDependenciesManager internal constructor(
         @TestOnly
         fun updateScriptDependenciesSynchronously(virtualFile: VirtualFile, project: Project) {
             with(getInstance(project)) {
-                val scriptDefinition = KotlinScriptDefinitionProvider.getInstance(project)!!.findScriptDefinition(virtualFile)!!
+                val scriptDefinition = ScriptDefinitionProvider.getInstance(project).findScriptDefinition(virtualFile)!!
                 cacheUpdater.updateSync(virtualFile, scriptDefinition)
                 cacheUpdater.notifyRootsChanged()
             }
         }
-
-        @TestOnly
-        fun reloadScriptDefinitions(project: Project) {
-            with(getInstance(project)) {
-                reloadScriptDefinitions()
-                cacheUpdater.clear()
-            }
-        }
     }
 }

idea/idea-core/src/org/jetbrains/kotlin/idea/core/script/ScriptDependenciesUpdater.kt

@@ -47,10 +47,10 @@ import kotlin.script.experimental.dependencies.AsyncDependenciesResolver
 import kotlin.script.experimental.dependencies.DependenciesResolver
 import kotlin.script.experimental.dependencies.ScriptDependencies
 
-internal class ScriptDependenciesUpdater(
+class ScriptDependenciesUpdater(
         private val project: Project,
         private val cache: ScriptDependenciesCache,
-        private val scriptDefinitionProvider: KotlinScriptDefinitionProvider
+        private val scriptDefinitionProvider: ScriptDefinitionProvider
 ) {
     private val requests = ConcurrentHashMap<String, ModStampedRequest>()
     private val contentLoader = ScriptContentLoader(project)

idea/idea-core/src/org/jetbrains/kotlin/idea/core/script/dependencies/KotlinScriptResolveScopeProvider.kt

@@ -21,28 +21,22 @@ import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.psi.ResolveScopeProvider
 import com.intellij.psi.search.GlobalSearchScope
 import org.jetbrains.kotlin.idea.core.script.ScriptDependenciesManager
+import org.jetbrains.kotlin.idea.core.script.StandardIdeScriptDefinition
 import org.jetbrains.kotlin.script.KotlinScriptDefinitionFromAnnotatedTemplate
 import org.jetbrains.kotlin.script.getScriptDefinition
 
 class KotlinScriptResolveScopeProvider : ResolveScopeProvider() {
-    companion object {
-        val USE_NULL_RESOLVE_SCOPE = "USE_NULL_RESOLVE_SCOPE"
-    }
 
     override fun getResolveScope(file: VirtualFile, project: Project): GlobalSearchScope? {
         val scriptDefinition = getScriptDefinition(file, project)
         // TODO: this should get this particular scripts dependencies
         return when {
             scriptDefinition == null -> null
-            // This is a workaround for completion in scripts and REPL to provide module dependencies
+        // This is a workaround for completion in scripts and REPL to provide module dependencies
             scriptDefinition.template == Any::class -> null
+            scriptDefinition is StandardIdeScriptDefinition -> null
             scriptDefinition is KotlinScriptDefinitionFromAnnotatedTemplate -> // TODO: should include the file itself
-                if (scriptDefinition.environment?.containsKey(USE_NULL_RESOLVE_SCOPE) ?: false) {
-                    null
-                }
-                else {
-                    ScriptDependenciesManager.getInstance(project).getAllScriptsClasspathScope()
-                }
+                ScriptDependenciesManager.getInstance(project).getAllScriptsClasspathScope()
             else -> null
         }
     }