Gradle, JS: Extract NpmResolutionManager

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/nodejs/NodeJsRootExtension.kt

@@ -1,11 +1,15 @@
 package org.jetbrains.kotlin.gradle.targets.js.nodejs
 
 import org.gradle.api.Project
+import org.jetbrains.kotlin.gradle.internal.isInIdeaSync
 import org.jetbrains.kotlin.gradle.logging.kotlinInfo
 import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJsCompilation
 import org.jetbrains.kotlin.gradle.targets.js.NpmVersions
+import org.jetbrains.kotlin.gradle.targets.js.npm.KotlinNpmResolutionManager
 import org.jetbrains.kotlin.gradle.targets.js.npm.NpmApi
+import org.jetbrains.kotlin.gradle.targets.js.npm.NpmDependency
 import org.jetbrains.kotlin.gradle.targets.js.npm.RequiresNpmDependencies
+import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinCompilationNpmResolution
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinRootNpmResolution
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinProjectNpmResolution
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolver.KotlinRootNpmResolver
@@ -86,70 +90,7 @@ open class NodeJsRootExtension(val rootProject: Project) {
     }
 
     val versions = NpmVersions()
-
+    internal val npmResolutionManager = KotlinNpmResolutionManager(this)
-    internal val resolutionState: ResolutionState
-        get() = _resolutionState
-
-    @Volatile
-    private var _resolutionState: ResolutionState = ResolutionState.Resolving(KotlinRootNpmResolver(this))
-
-    internal sealed class ResolutionState : ResolutionStateData {
-        class Resolving(val resolver: KotlinRootNpmResolver) : ResolutionState(), ResolutionStateData by resolver
-        class Resolved(val resolved: KotlinRootNpmResolution) : ResolutionState(), ResolutionStateData by resolved
-    }
-
-    interface ResolutionStateData {
-        val compilations: Collection<KotlinJsCompilation>
-    }
-
-    internal fun requireResolver(): KotlinRootNpmResolver =
-        (resolutionState as? ResolutionState.Resolving ?: error("NPM Dependencies already resolved and installed")).resolver
-
-    internal fun resolveIfNeeded(requireUpToDateReason: String? = null): KotlinRootNpmResolution {
-        val state0 = _resolutionState
-        val resolution = when (state0) {
-            is ResolutionState.Resolved -> state0.resolved
-            is ResolutionState.Resolving -> {
-                synchronized(this) {
-                    val state1 = _resolutionState
-                    when (state1) {
-                        is ResolutionState.Resolved -> state1.resolved
-                        is ResolutionState.Resolving -> state1.resolver.close().also {
-                            _resolutionState = ResolutionState.Resolved(it)
-                            if (requireUpToDateReason != null && !it.wasUpToDate) {
-                                error("NPM dependencies should be resolved $requireUpToDateReason")
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        return resolution
-    }
-
-    internal fun requireAlreadyResolved(project: Project, reason: String = ""): KotlinProjectNpmResolution =
-        resolveIfNeeded(reason)[project]
-
-    internal fun getAlreadyResolvedOrNull(project: Project): KotlinProjectNpmResolution? {
-        val state0 = resolutionState
-        return when (state0) {
-            is ResolutionState.Resolved -> state0.resolved[project]
-            is ResolutionState.Resolving -> null
-        }
-    }
-
-    internal fun checkRequiredDependencies(project: Project, target: RequiresNpmDependencies) {
-        val requestedTaskDependencies = requireAlreadyResolved(project, "before $target execution").taskRequirements
-        val targetRequired = requestedTaskDependencies[target]?.toSet() ?: setOf()
-
-        target.requiredNpmDependencies.forEach {
-            check(it in targetRequired) {
-                "${it.createDependency(project)} required by $target was not found resolved at the time of nodejs package manager call. " +
-                        "This may be caused by changing $target configuration after npm dependencies resolution."
-            }
-        }
-    }
 
     companion object {
         const val EXTENSION_NAME: String = "kotlinNodeJs"

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/KotlinNpmResolutionManager.kt

@@ -0,0 +1,189 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.targets.js.npm
+
+import org.gradle.api.Project
+import org.gradle.api.Task
+import org.jetbrains.kotlin.gradle.internal.isInIdeaSync
+import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJsCompilation
+import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension
+import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinCompilationNpmResolution
+import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinProjectNpmResolution
+import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinRootNpmResolution
+import org.jetbrains.kotlin.gradle.targets.js.npm.resolver.KotlinRootNpmResolver
+import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinNpmInstallTask
+
+/**
+ * # NPM resolution state manager
+ *
+ * ## Resolving process from Gradle
+ *
+ * **configuring**. Initial state. [NpmResolverPlugin] should be applied for each project
+ * that requires NPM resolution. This plugin should be applied only after kotlin plugin.
+ * When applied, [KotlinProjectNpmResolver] will be created for the corresponding project
+ * and will subscribe to all js compilations.
+ *
+ * **up-to-date-checked**. This state is compilation local (one compilation may be in up-to-date-checked
+ * state, while another may be steel in configuring state). New compilations may be added in this
+ * state, but compilations that are already up-to-date-checked cannot be changes.
+ * Initiated by calling [KotlinPackageJsonTask.producerInputs] getter (will be called by Gradle).
+ * [KotlinCompilationNpmResolver] will create and **resolve** aggregated compilation configuration,
+ * which consists of all related compilation configuration
+ * and npm tools configuration which contains all dependencies that is required for all enabled
+ * tasks related to this compilation. It is important to resolve this configuration inside particular
+ * project and not globally. Aggregated configuration will be analyzed for gradle internal dependencies
+ * (project depenencies), gradle external dependencies and npm dependencies. This collections will
+ * be treated as `packageJson` task inputs.
+ *
+ * **package-json-created**. This state also compilation local. Initiated by executing `packageJson`
+ * task for particular compilation.
+ *
+ * Note that package.json will be executed only for required compilations, while other may be missed.
+ *
+ * **installing**
+ *
+ * **installed**
+ *
+ *
+ *
+ * - resolving:
+ *   - [KotlinPackageJsonTask] task executed for each affected compilation:
+ *      - ensure that all needed package.json files created
+ *   - global [KotlinNpmInstallTask] executed:
+ *      - call package manager to sync node_modules
+ *
+ * Resolved state requested by calling [requireAlreadyInstalled] from this places:
+ *  - tasks that requires node_modules
+ *  - before calling [NpmProject.require]
+ *
+ * Also resolved state used when getting files of [NpmDependency]. In this case
+ * resolved project is not required. [NpmDependency] will not return any files if
+ * ... [requireAlreadyResolvedOrNullIfResolvingNow]
+ *
+ * Note that [KotlinPackageJsonTask] executed for required compilations only.
+ * [KotlinNpmInstallTask] may not be executed if none of package.json tasks executed.
+ * In this case [_resolutionState] will remain unclosed and will be closed
+ * on first [requireAlreadyInstalled] call with check that everything is already up-to-date.
+ *
+ *
+ *
+ * ## Resolving process during Idea import
+ */
+class KotlinNpmResolutionManager(val nodeJsSettings: NodeJsRootExtension) {
+    internal val resolutionState: ResolutionState
+        get() = _resolutionState
+
+    private val forceFullResolve: Boolean
+        get() = isInIdeaSync
+
+    @Volatile
+    private var _resolutionState: ResolutionState =
+        ResolutionState.Configuring(
+            KotlinRootNpmResolver(nodeJsSettings, forceFullResolve)
+        )
+
+    internal sealed class ResolutionState : ResolutionStateData {
+        class Configuring(val resolver: KotlinRootNpmResolver) : ResolutionState(), ResolutionStateData by resolver
+        class Installing(val resolver: KotlinRootNpmResolver) : ResolutionState(), ResolutionStateData by resolver
+        class Installed(val resolved: KotlinRootNpmResolution) : ResolutionState(), ResolutionStateData by resolved
+    }
+
+    interface ResolutionStateData {
+        val compilations: Collection<KotlinJsCompilation>
+    }
+
+    internal fun requireConfiguringState(): KotlinRootNpmResolver =
+        (resolutionState as? ResolutionState.Configuring ?: error("NPM Dependencies already resolved and installed")).resolver
+
+    /**
+     * @param requireUpToDateReason Check that project already resolved,
+     * or it is up-to-date but just not closed. Show given message if it is not.
+     * @param requireNotInstalled Check that project is not resolved
+     */
+    internal fun installIfNeeded(
+        requireUpToDateReason: String? = null,
+        requireNotInstalled: Boolean = false
+    ): KotlinRootNpmResolution {
+        fun alreadyResolved(resolution: KotlinRootNpmResolution): KotlinRootNpmResolution {
+            if (requireNotInstalled) error("Project already resolved")
+            return resolution
+        }
+
+        val state0 = _resolutionState
+        val resolution = when (state0) {
+            is ResolutionState.Installed -> alreadyResolved(state0.resolved)
+            is ResolutionState.Installing,
+            is ResolutionState.Configuring -> {
+                synchronized(this) {
+                    val state1 = _resolutionState
+                    when (state1) {
+                        is ResolutionState.Installed -> alreadyResolved(state1.resolved)
+                        is ResolutionState.Installing -> error("Resolution state sync failed")
+                        is ResolutionState.Configuring -> {
+                            val state2 =
+                                ResolutionState.Installing(state1.resolver)
+                            _resolutionState = state2
+                            state1.resolver.close().also {
+                                check(_resolutionState == state2) {
+                                    "Resolution state sync failed: $_resolutionState != $state2"
+                                }
+                                _resolutionState =
+                                    ResolutionState.Installed(it)
+                                if (requireUpToDateReason != null && !it.wasUpToDate) {
+                                    error("NPM dependencies should be resolved $requireUpToDateReason")
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        return resolution
+    }
+
+    internal fun requireAlreadyInstalled(project: Project, reason: String = ""): KotlinProjectNpmResolution =
+        installIfNeeded(requireUpToDateReason = reason)[project]
+
+    internal fun getNpmDependencyResolvedCompilation(npmDependency: NpmDependency): KotlinCompilationNpmResolution {
+        val project = npmDependency.project
+
+        val resolvedProject =
+            if (forceFullResolve) {
+                installIfNeeded()[project]
+            } else {
+                // may return null only during npm resolution
+                // (it can be called since NpmDependency added to configuration that
+                // requires resolve to build package.json, in this case we should just skip this call)
+                val state0 = resolutionState
+                when (state0) {
+                    is ResolutionState.Installed -> state0.resolved[project]
+                    is ResolutionState.Installing -> null
+                    is ResolutionState.Configuring -> error("Cannot use NpmDependency before :kotlinNpmInstall task execution")
+                }
+            }
+
+        return when (resolvedProject) {
+            null -> null
+            else -> resolvedProject.npmProjectsByNpmDependency[npmDependency] ?: error("NPM project resolved without $this")
+        }
+    }
+
+    internal fun <T> checkRequiredDependencies(task: T)
+            where T : RequiresNpmDependencies,
+                  T : Task {
+        val project = task.project
+        val requestedTaskDependencies = requireAlreadyInstalled(project, "before $task execution").taskRequirements
+        val targetRequired = requestedTaskDependencies[task]?.toSet() ?: setOf()
+
+        task.requiredNpmDependencies.forEach {
+            check(it in targetRequired) {
+                "${it.createDependency(project)} required by $task was not found resolved at the time of nodejs package manager call. " +
+                        "This may be caused by changing $task configuration after npm dependencies resolution."
+            }
+        }
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmDependency.kt

@@ -15,9 +15,7 @@ import org.gradle.api.internal.artifacts.ResolvableDependency
 import org.gradle.api.internal.artifacts.dependencies.SelfResolvingDependencyInternal
 import org.gradle.api.tasks.TaskDependency
 import org.gradle.internal.component.local.model.DefaultLibraryBinaryIdentifier
-import org.jetbrains.kotlin.gradle.internal.isInIdeaSync
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinCompilationNpmResolution
-import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinProjectNpmResolution
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootPlugin
 import java.io.File
 
@@ -83,23 +81,14 @@ data class NpmDependency(
         }
     }
 
+    // may return null only during npm resolution
+    // (it can be called since NpmDependency added to configuration that
+    // requires resolve to build package.json, in this case we should just skip this call)
     private fun resolveProject(): KotlinCompilationNpmResolution? {
         val nodeJs = NodeJsRootPlugin.apply(project.rootProject)
-
+        return nodeJs.npmResolutionManager.getNpmDependencyResolvedCompilation(this)
-        // may return null only while resolving configuration in ConfigurationVisitor.visit
-        val resolvedProject =
-            if (isInIdeaSync) nodeJs.resolveIfNeeded()[project]
-            else nodeJs.getAlreadyResolvedOrNull(project)
-
-        return when (resolvedProject) {
-            null -> null
-            else -> findIn(resolvedProject) ?: error("NPM project resolved without $this")
-        }
     }
 
-    private fun findIn(npmProjects: KotlinProjectNpmResolution) =
-        npmProjects.npmProjectsByNpmDependency[this]
-
     val key: String = if (org == null) name else "@$org/$name"
 
     override fun toString() = "$key: $version"

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmProject.kt

@@ -72,7 +72,7 @@ open class NpmProject(val compilation: KotlinJsCompilation) {
      * Require [request] nodejs module and return canonical path to it's main js file.
      */
     fun require(request: String): String {
-        nodeJs.requireAlreadyResolved(project)
+        nodeJs.npmResolutionManager.requireAlreadyInstalled(project)
         return modules.require(request)
     }
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmResolverPlugin.kt

@@ -7,15 +7,13 @@ package org.jetbrains.kotlin.gradle.targets.js.npm
 
 import org.gradle.api.Plugin
 import org.gradle.api.Project
-import org.jetbrains.kotlin.gradle.dsl.kotlinExtension
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootPlugin
-import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinPackageJsonTask
 
 class NpmResolverPlugin : Plugin<Project> {
     override fun apply(target: Project) {
         val project = target
         val nodeJs = NodeJsRootPlugin.apply(project.rootProject)
-        nodeJs.requireResolver().addProject(project)
+        nodeJs.npmResolutionManager.requireConfiguringState().addProject(project)
     }
 
     companion object {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/resolved/KotlinRootNpmResolution.kt

@@ -7,13 +7,13 @@ package org.jetbrains.kotlin.gradle.targets.js.npm.resolved
 
 import org.gradle.api.Project
 import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJsCompilation
-import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension
+import org.jetbrains.kotlin.gradle.targets.js.npm.KotlinNpmResolutionManager
 
 class KotlinRootNpmResolution(
     val wasUpToDate: Boolean,
     val rootProject: Project,
     private val projects: Map<Project, KotlinProjectNpmResolution>
-) : NodeJsRootExtension.ResolutionStateData {
+) : KotlinNpmResolutionManager.ResolutionStateData {
     operator fun get(project: Project) = projects[project] ?: KotlinProjectNpmResolution.empty(project)
 
     override val compilations: Collection<KotlinJsCompilation>

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/resolver/KotlinCompilationNpmResolver.kt

@@ -49,31 +49,28 @@ internal class KotlinCompilationNpmResolver(
     }
 
     private var closed = false
-    private var resolveCalled = false
+    private var resolution: KotlinCompilationNpmResolution? = null
 
-    private val resolution: KotlinCompilationNpmResolution by lazy {
+    @Synchronized
-        check(!closed) { "$this already closed" }
-        resolveCalled = true
-        packageJsonProducer.createPackageJson()
-    }
-
-    /**
-     * Called from packageJson task
-     */
     fun resolve() {
         check(!closed) { "$this already closed" }
-        check(!resolveCalled) { "$this already resolved" }
+        check(resolution == null) { "$this already resolved" }
 
-        // call getter
+        resolution = packageJsonProducer.createPackageJson()
-        resolution
     }
 
+    @Synchronized
+    fun getResolutionOrResolveIfForced(): KotlinCompilationNpmResolution? {
+        if (resolver.forceFullResolve && resolution == null) resolve()
+        return resolution
+    }
+
+    @Synchronized
     fun close(): KotlinCompilationNpmResolution? {
         check(!closed) { "$this already closed" }
-        val result = resolution
+        val resolution = getResolutionOrResolveIfForced()
         closed = true
-
+        return resolution
-        return result
     }
 
     private fun createAggregatedConfiguration(): Configuration {
@@ -205,10 +202,8 @@ internal class KotlinCompilationNpmResolver(
 
         fun createPackageJson(): KotlinCompilationNpmResolution {
             val resolvedInternalDependencies = internalDependencies.map {
-//                check(it.resolveCalled) {
+                it.getResolutionOrResolveIfForced()
-//                    "Unresolved dependent npm package: ${this@KotlinCompilationNpmResolver} -> $it"
+                    ?: error("Unresolved dependent npm package: ${this@KotlinCompilationNpmResolver} -> $it")
-//                }
-                it.resolution
             }
             val importedExternalGradleDependencies = externalGradleDependencies.mapNotNull {
                 resolver.gradleNodeModules.get(it.dependency, it.artifact)

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/resolver/KotlinRootNpmResolver.kt

@@ -22,7 +22,10 @@ import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinRootNpmResoluti
  * For external gradle modules, fake npm packages will be created and added to `package.json`
  * as path to directory.
  */
-internal class KotlinRootNpmResolver internal constructor(val nodeJs: NodeJsRootExtension) : NodeJsRootExtension.ResolutionStateData {
+internal class KotlinRootNpmResolver internal constructor(
+    val nodeJs: NodeJsRootExtension,
+    val forceFullResolve: Boolean
+) : KotlinNpmResolutionManager.ResolutionStateData {
     val rootProject: Project
         get() = nodeJs.rootProject
 
@@ -68,7 +71,9 @@ internal class KotlinRootNpmResolver internal constructor(val nodeJs: NodeJsRoot
         check(!closed)
         closed = true
 
-        val projectResolutions = projectResolvers.values.map { it.close() }.associateBy { it.project }
+        val projectResolutions = projectResolvers.values
+            .map { it.close() }
+            .associateBy { it.project }
         val allNpmPackages = projectResolutions.values.flatMap { it.npmProjects }
 
         gradleNodeModules.close()

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/tasks/KotlinNpmInstallTask.kt

@@ -23,7 +23,7 @@ open class KotlinNpmInstallTask : DefaultTask() {
     @Suppress("unused")
     @get:InputFiles
     val packageJsonFiles: Collection<File>
-        get() = nodeJs.resolutionState.compilations.map { it.npmProject.packageJsonFile }
+        get() = nodeJs.npmResolutionManager.resolutionState.compilations.map { it.npmProject.packageJsonFile }
 
     @get:OutputFile
     val yarnLock: File
@@ -31,7 +31,7 @@ open class KotlinNpmInstallTask : DefaultTask() {
 
     @TaskAction
     fun resolve() {
-        nodeJs.resolveIfNeeded()
+        nodeJs.npmResolutionManager.installIfNeeded(requireNotInstalled = true)
     }
 
     companion object {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/tasks/KotlinPackageJsonTask.kt

@@ -25,7 +25,7 @@ open class KotlinPackageJsonTask : DefaultTask() {
     private lateinit var compilation: KotlinJsCompilation
 
     private val compilationResolver
-        get() = nodeJs.requireResolver()[project][compilation]
+        get() = nodeJs.npmResolutionManager.requireConfiguringState()[project][compilation]
 
     private val producer: KotlinCompilationNpmResolver.PackageJsonProducer
         get() = compilationResolver.packageJsonProducer

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/testing/KotlinJsTest.kt

@@ -93,7 +93,7 @@ open class KotlinJsTest : KotlinTest(), RequiresNpmDependencies {
     }
 
     override fun executeTests() {
-        nodeJs.checkRequiredDependencies(project, this)
+        nodeJs.npmResolutionManager.checkRequiredDependencies(this)
         super.executeTests()
     }
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/webpack/KotlinWebpack.kt

@@ -127,7 +127,7 @@ open class KotlinWebpack : DefaultTask(), RequiresNpmDependencies {
 
     @TaskAction
     fun doExecute() {
-        nodeJs.checkRequiredDependencies(project, this)
+        nodeJs.npmResolutionManager.checkRequiredDependencies(this)
 
         val runner = createRunner()
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnBasics.kt

@@ -32,7 +32,6 @@ abstract class YarnBasics : NpmApi {
             exec.args = listOf(yarnEnv.home.resolve("bin/yarn.js").absolutePath) + args
             exec.workingDir = dir
         }
-
     }
 
     protected fun yarnLockReadTransitiveDependencies(