gradle.kts: docs

2020-05-06 12:14:27 +03:00
parent d6adab785a
commit e376a04b5a
6 changed files with 181 additions and 24 deletions
@@ -25,6 +25,21 @@ import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.scripting.definitions.ScriptDefinition
 import org.jetbrains.kotlin.scripting.resolve.ScriptCompilationConfigurationWrapper

+/**
+ * The [CompositeScriptConfigurationManager] will provide redirection of [ScriptConfigurationManager] calls to the
+ * custom [ScriptingSupport] or [DefaultScriptingSupport] if that script lack of custom [ScriptingSupport].
+ *
+ * The [ScriptConfigurationManager] is implemented by caching all scripts using the [ScriptClassRootsCache].
+ * The [ScriptClassRootsCache] is always available and never blacked by the writer, as all writes occurs
+ * using the copy-on-write strategy.
+ *
+ * This cache are loaded on start and will be updating asynchronously using the [updater].
+ * Sync updates still my be occurred from the [DefaultScriptingSupport].
+ *
+ * We are also watching all script documents:
+ * [notifier] will call first applicable [ScriptChangesNotifier.listeners] when editor is activated or document changed.
+ * Listener should do something to invalidate configuration and schedule reloading.
+ */
 class CompositeScriptConfigurationManager(val project: Project) : ScriptConfigurationManager {
    @Suppress("unused")
    private val notifier = ScriptChangesNotifier(project)
@@ -36,6 +36,60 @@ import java.util.concurrent.locks.ReentrantLock
 import kotlin.concurrent.withLock
 import kotlin.script.experimental.api.ScriptDiagnostic

+/**
+ * Standard implementation of scripts configuration loading and caching.
+ *
+ * ## Loading initiation
+ *
+ * [getOrLoadConfiguration] will be called when we need to show or analyze some script file.
+ *
+ * As described in [DefaultScriptingSupportBase], configuration may be loaded from [cache]
+ * or [reloadOutOfDateConfiguration] will be called on [cache] miss.
+ *
+ * There are 2 tiers [cache]: memory and FS. For now FS cache implemented by [ScriptConfigurationLoader]
+ * because we are not storing classpath roots yet. As a workaround cache.all() will return only memory
+ * cached configurations.  So, for now we are indexing roots that loaded from FS with
+ * default [reloadOutOfDateConfiguration] mechanics.
+ *
+ * Also, [ensureConfigurationUpToDate] may be called from [UnusedSymbolInspection]
+ * to ensure that configuration of all scripts containing some symbol are up-to-date or try load it in sync.
+ * Note: it makes sense only in case of "auto apply" mode and sync loader, in other cases all symbols just
+ * will be treated as used.
+ *
+ * ## Loading
+ *
+ * When requested, configuration will be loaded using first applicable [loaders].
+ * It can work synchronously or asynchronously.
+ *
+ * Synchronous loader will be called just immediately. Despite this, its result may not be applied immediately,
+ * see next section for details.
+ *
+ * Asynchronous loader will be called in background thread (by [BackgroundExecutor]).
+ *
+ * ## Applying
+ *
+ * By default loaded configuration will *not* be applied immediately. Instead, we show in editor notification
+ * that suggests user to apply changed configuration. This was done to avoid sporadically starting indexing of new roots,
+ * which may happens regularly for large Gradle projects.
+ *
+ * Notification will be displayed when configuration is going to be updated. First configuration will be loaded
+ * without notification.
+ *
+ * This behavior may be disabled by enabling "auto reload" in project settings.
+ * When enabled, all loaded configurations will be applied immediately, without any notification.
+ *
+ * ## Concurrency
+ *
+ * Each files may be in on of this state:
+ * - scriptDefinition is not ready
+ * - not loaded
+ * - up-to-date
+ * - invalid, in queue (in [BackgroundExecutor] queue)
+ * - invalid, loading
+ * - invalid, waiting for apply
+ *
+ * [reloadOutOfDateConfiguration] guard this states. See it's docs for more details.
+ */
 class DefaultScriptingSupport(manager: CompositeScriptConfigurationManager) : DefaultScriptingSupportBase(manager) {
    // TODO public for tests
    val backgroundExecutor: BackgroundExecutor =
@@ -286,7 +340,19 @@ class DefaultScriptingSupport(manager: CompositeScriptConfigurationManager) : De
    }
 }

-abstract class DefaultScriptingSupportBase(val manager: CompositeScriptConfigurationManager) : ScriptingSupport() {
+/**
+ * Abstraction for [DefaultScriptingSupportBase] based [cache] and [reloadOutOfDateConfiguration].
+ * Among this two methods concrete implementation should provide script changes listening.
+ *
+ * Basically all requests routed to [cache]. If there is no entry in [cache] or it is considered out-of-date,
+ * then [reloadOutOfDateConfiguration] will be called, which, in turn, should call [setAppliedConfiguration]
+ * immediately or in some future  (e.g. after user will click "apply context" or/and configuration will
+ * be calculated by some background thread).
+ *
+ * [ScriptClassRootsCache] will be calculated based on [cache]d configurations.
+ * Every change in [cache] will invalidate [ScriptClassRootsCache] cache.
+ */
+abstract class DefaultScriptingSupportBase(val manager: CompositeScriptConfigurationManager) {
    val project: Project
        get() = manager.project

@@ -0,0 +1,75 @@
+# Kotlin Gradle build scripts support.
+
+## Why custom Scripting support
+
+The scripting support for Gradle becomes tricky since the default one was designed for a little bit another case: each script becomes configured individually on first opening/usage and loading supposes to be instant. 
+
+Unfortunately, this is not working well for the Gradle case: the script configuration gathering is long operation since the Gradle configuration phase required to be executed to get it. For large projects, it may take from 5 seconds for hot daemon and to minutes for initial project loading. It may download dependencies, compile and build buildSrc project and all these things will be done on first script usage, even when we need just to know is such word in reference to our symbol when we are calling "Find usages". Among that, Gradle is not able to gather individual script configuration and always provides configuration for all scripts.
+
+Taking this in mind, it becomes better to have this strategy for Gradle scripts:
+- The user performing Gradle project importing/reimporting by executing Gradle configuration phase
+- We have all Gradle scripts configured
+- We can save this configuration to FS cache which can be persisted between IntelliJ restarts.
+
+## More details about what we already have
+
+[ScriptConfigurationManager] service already existed for providing and updating scripts configuration. It was working by loading scripts configuration on the first opening, storing in-memory cache, and FS cache. 
+
+It also reloads it after each typing (with some throttling) and notifies the user if the configuration is changed with a suggestion to apply a new configuration. We may apply this configuration automatically, but it may introduce indexing which breaks UX (some action becomes unavailable during indexing). To leverage this situation, we are showing notification instead and indexing will occur only after an explicit user action. We can do this automatically if no new source and classes are introduced in dependencies, but this is not implemented yet.
+
+Note: as we don't know in general when some script change causes script configuration change, we always run configuration reloading and comparing new configuration with the old one. If we can know that some particular change will affect script configuration, we can show notification before starting expensive loading, but this requires some knowledge of the script, so should be implemented by the script definition author. [ScriptConfigurationManager] provides ability to implement up-to-date stamp by extension point. It is implemented for the older Gradle versions, see "DefaultScriptingSupport extensions for older Gradle versions" below.
+
+The manager also creates a list of all classes and sources of script dependencies to properly support analyzing of scripts. All these calculations are cached and invalidated on each change in script configurations.
+
+Sometimes this loading may occur in the background, sometimes in sync. We also may apply configuration automatically in some cases (first loading, loading in sync, etc). All that introduces complex states intersection which is tricky to support. Adding another layer of complexity to that implementation is not a way to go.
+
+## Custom scripting support
+
+After some experiments and several iterations, the final idea is to provide the ability to create custom mechanics for managing the scripts configurations without breaking default scripting support.
+
+As we already need a cache of all known scripts, the union of its classpath, and source path which is rebuilt from the scratch on any change, we may just provide the ability to populate this cache with custom content on each invalidation. That the main idea behind custom scripting support.
+
+The other idea is to don't make any blocking work synchronously while getting some configuration. The cache should be always ready for unblocking reading. All updating operation is asynchronous and non-blocking thanks to copy-on-write strategy.
+
+So, it should work in that way:
+- Custom scripting support should manage the scripts configurations on its own (with own in-memory/fs cache).
+- It should implement a method that will gather all script configuration providers and populate the classpath/sourcepath union that is used to get files for indexing.
+- It may implement ScriptChangeListener extension point to do some invalidation on document change events.
+- It should call `updater.update { ... updater.invalidate() ...  }` when the new script configurations are required. The manager will schedule an asynchronous update in the background.
+- It is not able to load new configurations lazily on the first request (it is still possible to calculate script configuration lazily, but file name should be already registered)
+
+The [CompositeScriptConfigurationManager] will provide redirection of [ScriptConfigurationManager] calls to the custom [ScriptingSupport] or [DefaultScriptingSupport].
+
+## Gradle CustomScriptingSupport
+
+`GradleBuildRootsManager` implementing `ScriptingSupport` described above.
+
+There are many Gradle builds can be linked with a single IntelliJ project (don't confuse with the Gradle project and included build). This complicates implementation a bit, as we should manage this builds separately: each of them may have its own script definitions, Gradle version and java home. Typically, the IntelliJ project has no more than one [GradleBuildRoot].
+
+`GradleBuildRootsManager` actually managing linked Gradle builds. See it's KDoc for more details.
+
+The script configuration is stored in FS by using the IntelliJ VFS file attributes.
+
+Note that we can provide custom scripting support only for projects that using Gradle 6.0 and later, as gathering script models unavailable in older Gradle versions. `GradleBuildRootsManager` falling back to default scripting support for such linked Gradle builds. It is extended through: see "DefaultScriptingSupport extensions for older Gradle versions" for more details.
+
+## Watching files states across IntelliJ restarts
+
+To have consistent sate of scripts, we should also be aware of external script changes. This is achieved by watching files using the IntelliJ VFS events. [GradleScriptInputsWatcher] is responsible for that.
+
+The first tricky part is that scripts are depending on each other: so, when one script is changed, we actually should invalidate all other scripts as we don't know dependencies between them (Gradle will provide this information later, but it is not yet implemented). Actually, we should know the last modified timestamp of all scripts excepting a particular one. This can be achieved by storing timestamps of two last modified files. [LastModifiedFiles] utility is responsible for that. This is not multiple linked Gradle builds aware yet (currently there is only one global state), but it will be fixed shortly.
+
+Another tricky part is that we should track only scripts belong the Gradle project and should ignore all other `*.gradle.kts` files (in `testData` for example). This is achieved by storing Gradle project roots, as scripts can be exactly near Gradle project roots (excepting included and precompiled scripts which are not fully supported yet). This can be gathered from the Gradle project import information or from GradleProjectSettings when the import has not occurred yet. `GradleScriptInputsWatcher.cachedGradleProjectsRoots` is responsible for that.
+
+## Out of project Gradle scripts
+
+There are scripts that are not linked to any import Gradle project. In `testData` for example. Currently, we are suggesting to import Gradle project for such scripts, but we also may implement standalone Gradle scripts supporting using the `DefaultScriptingSupport` for explicitly listed scripts. We can implement this in the future if it will be valuable for users.
+
+## DefaultScriptingSupport extensions for older Gradle versions
+
+For Gradle versions before 6.0, we are still falling back to the [DefaultScriptingSupport] with some extensions.
+
+We are showing notification before loading as we knew what changes will case it. This is done by:
+- implementing `org.jetbrains.kotlin.scripting.idea.listener` extension point and calling `suggestToUpdateConfigurationIfOutOfDate` instead of `ensureUpToDatedConfigurationSuggested` on document changes.
+- implementing `org.jetbrains.kotlin.scripting.idea.loader` extension point and overriding `getInputsStamp`
+
+We are also managing out of project scripts the same way as it is done in `GradleBuildRootsManager`.
@@ -16,18 +16,23 @@ import org.jetbrains.kotlin.scripting.resolve.VirtualFileScriptSource
 import java.io.File

 /**
- * See [GradleBuildRootsManager]
+ * [GradleBuildRoot] is a linked gradle build (don't confuse with gradle project and included build).
+ * Each [GradleBuildRoot] may have it's own Gradle version, Java home and other settings.
+ *
+ * Typically, IntelliJ project have no more than one [GradleBuildRoot].
+ *
+ * See [GradleBuildRootsManager] for more details.
 */
 sealed class GradleBuildRoot {
    /**
-     * Add Gradle Project
-     * for other scripts too
-     *
-     * precompiled script
-     * may be also included scripts not returned by gradle: todo proper notification
+     * The script not related to any Gradle build that is linked to IntelliJ Project,
+     * or we cannot known what is it
     */
-    class Unlinked() : GradleBuildRoot()
+    class Unlinked : GradleBuildRoot()

+    /**
+     * Linked project, that may be itself: [Legacy], [New] or [Imported].
+     */
    abstract class Linked : GradleBuildRoot() {
        @Volatile
        var importing = false
@@ -38,25 +43,19 @@ sealed class GradleBuildRoot {
    }

    /**
-     * Notification: please update to Gradle 6.0
-     * default loader, cases:
-     * - not loaded: Notification: Load configration to get code insights
-     * - loaded, not up-to-date: Notifaction: Reload configuraiton
-     * - loaded, up-to-date: Nothing
+     * Gradle build with old Gradle version (<6.0)
     */
    class Legacy(override val pathPrefix: String) : Linked()

    /**
-     * not imported:
-     *  Notification: Import Gradle project to get code insights
-     * during import:
-     * - disable action on importing. todo: don't miss failed import
-     * - pause analyzing, todo: change status text to: importing gradle project
+     * Linked but not yet imported Gradle build.
     */
    class New(override val pathPrefix: String) : Linked()

-    // precompiled scripts not detected by gradle
-
+    /**
+     * Imported Gradle build.
+     * Each imported build have info about all of it's Kotlin Build Scripts.
+     */
    class Imported(
        val project: Project,
        val dir: VirtualFile,
@@ -11,9 +11,9 @@ class GradleBuildRootIndex {
    private val log = logger<GradleBuildRootIndex>()

    private val byWorkingDir = HashMap<String, GradleBuildRoot.Linked>()
-    val byProjectDir = HashMap<String, GradleBuildRoot.Linked>()
+    private val byProjectDir = HashMap<String, GradleBuildRoot.Linked>()

-    val values: Collection<GradleBuildRoot>
+    val list: Collection<GradleBuildRoot>
        get() = byWorkingDir.values

    @Synchronized
@@ -33,7 +33,9 @@ import java.nio.file.Paths

 /**
 * [GradleBuildRoot] is a linked gradle build (don't confuse with gradle project and included build).
- * Each [GradleBuildRoot] may have it's own Gradle version, java home and other settings.
+ * Each [GradleBuildRoot] may have it's own Gradle version, Java home and other settings.
+ *
+ * Typically, IntelliJ project have no more than one [GradleBuildRoot].
 *
 * This manager allows to find related Gradle build by the Gradle Kotlin script file path.
 * Each imported build have info about all of it's Kotlin Build Scripts.
@@ -88,7 +90,7 @@ class GradleBuildRootsManager(val project: Project) : ScriptingSupport.Provider(
    }

    override fun collectConfigurations(builder: ScriptClassRootsCache.Builder) {
-        roots.values.forEach { root ->
+        roots.list.forEach { root ->
            if (root is GradleBuildRoot.Imported) {
                root.collectConfigurations(builder)
            }