Gradle, JS, NpmResolutionManager: cleanup and document states

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/KotlinNpmResolutionManager.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.gradle.targets.js.npm
 
+import org.gradle.api.Incubating
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.jetbrains.kotlin.gradle.internal.isInIdeaSync
@@ -14,14 +15,14 @@ import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinCompilationNpmR
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinProjectNpmResolution
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinRootNpmResolution
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolver.KotlinRootNpmResolver
-import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinNpmInstallTask
+import java.io.File
 
 /**
  * # NPM resolution state manager
  *
  * ## Resolving process from Gradle
  *
- * **configuring**. Initial state. [NpmResolverPlugin] should be applied for each project
+ * **configuring**. Global initial state. [NpmResolverPlugin] should be applied for each project
  * that requires NPM resolution. This plugin should be applied only after kotlin plugin.
  * When applied, [KotlinProjectNpmResolver] will be created for the corresponding project
  * and will subscribe to all js compilations.
@@ -43,67 +44,67 @@ import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinNpmInstallTask
  *
  * Note that package.json will be executed only for required compilations, while other may be missed.
  *
- * **installing**
+ * **installed**. Global final state. Initiated by executing global `kotlinNpmInstall` task.
- *
+ * All created package.json files will be gathered and package manager will be executed.
- * **installed**
+ * Package manager will create lock file, that will be parsed for transitive npm dependencies
- *
+ * that will be added to the root [NpmDependency] objects. `kotlinNpmInstall` task may be up-to-date.
- *
+ * In this case, installed state will be reached by first call of [installIfNeeded] without exeucting
- *
+ * package manager.
- * - resolving:
- *   - [KotlinPackageJsonTask] task executed for each affected compilation:
- *      - ensure that all needed package.json files created
- *   - global [KotlinNpmInstallTask] executed:
- *      - call package manager to sync node_modules
- *
- * Resolved state requested by calling [requireAlreadyInstalled] from this places:
- *  - tasks that requires node_modules
- *  - before calling [NpmProject.require]
- *
- * Also resolved state used when getting files of [NpmDependency]. In this case
- * resolved project is not required. [NpmDependency] will not return any files if
- * ... [requireAlreadyResolvedOrNullIfResolvingNow]
- *
- * Note that [KotlinPackageJsonTask] executed for required compilations only.
- * [KotlinNpmInstallTask] may not be executed if none of package.json tasks executed.
- * In this case [_resolutionState] will remain unclosed and will be closed
- * on first [requireAlreadyInstalled] call with check that everything is already up-to-date.
  *
+ * Resolution will be used from [NpmDependency] by calling [getNpmDependencyResolvedCompilation].
+ * Also resolution will be checked before calling [NpmProject.require] and executing any task
+ * that requires npm dependencies or node_modules.
  *
+ * Also user can call [requireInstalled] to get resolution info.
  *
  * ## Resolving process during Idea import
+ *
+ * In this case [forceFullResolve] will be set, and all compilations will be resolved
+ * even without `packageJson` task execution.
+ *
+ * During building gradle project model, all [NpmDependency] will be asked for there files,
+ * and [getNpmDependencyResolvedCompilation] will be called. In the [forceFullResolve] mode
+ * project will be fully resolved: all `package.json` files will be created, and package manager
+ * will be called. We are manually tracking package.json files contents to avoid calling package manager
+ * is nothing was changes.
+ *
+ * Note that in this case resolution process will be performed outside of task execution.
  */
 class KotlinNpmResolutionManager(val nodeJsSettings: NodeJsRootExtension) {
-    internal val resolutionState: ResolutionState
-        get() = _resolutionState
-
     private val forceFullResolve: Boolean
         get() = isInIdeaSync
 
     @Volatile
-    private var _resolutionState: ResolutionState =
+    private var state: ResolutionState =
         ResolutionState.Configuring(
             KotlinRootNpmResolver(nodeJsSettings, forceFullResolve)
         )
 
-    internal sealed class ResolutionState : ResolutionStateData {
+    internal sealed class ResolutionState {
-        class Configuring(val resolver: KotlinRootNpmResolver) : ResolutionState(), ResolutionStateData by resolver
+        class Configuring(val resolver: KotlinRootNpmResolver) : ResolutionState()
-        class Installing(val resolver: KotlinRootNpmResolver) : ResolutionState(), ResolutionStateData by resolver
+        class Installed(val resolved: KotlinRootNpmResolution) : ResolutionState()
-        class Installed(val resolved: KotlinRootNpmResolution) : ResolutionState(), ResolutionStateData by resolved
     }
 
-    interface ResolutionStateData {
+    @Incubating
-        val compilations: Collection<KotlinJsCompilation>
+    fun requireInstalled() = (state as ResolutionState.Installed).resolved
-    }
 
     internal fun requireConfiguringState(): KotlinRootNpmResolver =
-        (resolutionState as? ResolutionState.Configuring ?: error("NPM Dependencies already resolved and installed")).resolver
+        (this.state as? ResolutionState.Configuring ?: error("NPM Dependencies already resolved and installed")).resolver
+
+    internal fun install() = installIfNeeded(requireNotInstalled = true)
+
+    internal fun requireAlreadyInstalled(project: Project, reason: String = ""): KotlinProjectNpmResolution =
+        installIfNeeded(requireUpToDateReason = reason)[project]
+
+    internal val packageJsonFiles: Collection<File>
+        get() = (state as ResolutionState.Configuring).resolver.compilations.map { it.npmProject.packageJsonFile }
 
     /**
      * @param requireUpToDateReason Check that project already resolved,
      * or it is up-to-date but just not closed. Show given message if it is not.
      * @param requireNotInstalled Check that project is not resolved
      */
-    internal fun installIfNeeded(
+    private fun installIfNeeded(
         requireUpToDateReason: String? = null,
         requireNotInstalled: Boolean = false
     ): KotlinRootNpmResolution {
@@ -112,29 +113,23 @@ class KotlinNpmResolutionManager(val nodeJsSettings: NodeJsRootExtension) {
             return resolution
         }
 
-        val state0 = _resolutionState
+        val state0 = this.state
         val resolution = when (state0) {
             is ResolutionState.Installed -> alreadyResolved(state0.resolved)
-            is ResolutionState.Installing,
             is ResolutionState.Configuring -> {
                 synchronized(this) {
-                    val state1 = _resolutionState
+                    val state1 = this.state
                     when (state1) {
                         is ResolutionState.Installed -> alreadyResolved(state1.resolved)
-                        is ResolutionState.Installing -> error("Resolution state sync failed")
                         is ResolutionState.Configuring -> {
-                            val state2 =
+                            val upToDate = nodeJsSettings.npmInstallTask.state.upToDate
-                                ResolutionState.Installing(state1.resolver)
+                            if (requireUpToDateReason != null && !upToDate) {
-                            _resolutionState = state2
+                                error("NPM dependencies should be resolved $requireUpToDateReason")
-                            state1.resolver.close().also {
+                            }
-                                check(_resolutionState == state2) {
+
-                                    "Resolution state sync failed: $_resolutionState != $state2"
+                            val skipPackageManager = upToDate && !forceFullResolve
-                                }
+                            state1.resolver.close(skipPackageManager).also {
-                                _resolutionState =
+                                this.state = ResolutionState.Installed(it)
-                                    ResolutionState.Installed(it)
-                                if (requireUpToDateReason != null && !it.wasUpToDate) {
-                                    error("NPM dependencies should be resolved $requireUpToDateReason")
-                                }
                             }
                         }
                     }
@@ -145,9 +140,6 @@ class KotlinNpmResolutionManager(val nodeJsSettings: NodeJsRootExtension) {
         return resolution
     }
 
-    internal fun requireAlreadyInstalled(project: Project, reason: String = ""): KotlinProjectNpmResolution =
-        installIfNeeded(requireUpToDateReason = reason)[project]
-
     internal fun getNpmDependencyResolvedCompilation(npmDependency: NpmDependency): KotlinCompilationNpmResolution {
         val project = npmDependency.project
 
@@ -158,18 +150,14 @@ class KotlinNpmResolutionManager(val nodeJsSettings: NodeJsRootExtension) {
                 // may return null only during npm resolution
                 // (it can be called since NpmDependency added to configuration that
                 // requires resolve to build package.json, in this case we should just skip this call)
-                val state0 = resolutionState
+                val state0 = state
                 when (state0) {
                     is ResolutionState.Installed -> state0.resolved[project]
-                    is ResolutionState.Installing -> null
                     is ResolutionState.Configuring -> error("Cannot use NpmDependency before :kotlinNpmInstall task execution")
                 }
             }
 
-        return when (resolvedProject) {
+        return resolvedProject.npmProjectsByNpmDependency[npmDependency] ?: error("NPM project resolved without $this")
-            null -> null
-            else -> resolvedProject.npmProjectsByNpmDependency[npmDependency] ?: error("NPM project resolved without $this")
-        }
     }
 
     internal fun <T> checkRequiredDependencies(task: T)

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmApi.kt

@@ -16,14 +16,10 @@ interface NpmApi {
 
     fun resolveProject(resolvedNpmProject: KotlinCompilationNpmResolution)
 
-    enum class Result {
-        upToDate, executed
-    }
-
     fun resolveRootProject(
         rootProject: Project,
         subProjects: Collection<KotlinCompilationNpmResolution>
-    ): Result
+    )
 
     companion object {
         fun resolveOperationDescription(packageManagerTitle: String): String =

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/PackageJsonUpToDateCheck.kt

@@ -1,19 +1,18 @@
 /*
- * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * that can be found in the license/LICENSE.txt file.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
-package org.jetbrains.kotlin.gradle.targets.js.yarn
+package org.jetbrains.kotlin.gradle.targets.js.npm
 
 import org.gradle.api.Project
 import org.gradle.api.internal.project.ProjectInternal
 import org.gradle.internal.hash.FileHasher
 import org.jetbrains.kotlin.daemon.common.toHexString
 import org.jetbrains.kotlin.gradle.internal.ensureParentDirsCreated
-import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProject
 import java.io.File
 
-class YarnUpToDateCheck(val npmProject: NpmProject) {
+class PackageJsonUpToDateCheck(val npmProject: NpmProject) {
     val project: Project
         get() = npmProject.project
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/resolved/KotlinRootNpmResolution.kt

@@ -10,12 +10,8 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJsCompilation
 import org.jetbrains.kotlin.gradle.targets.js.npm.KotlinNpmResolutionManager
 
 class KotlinRootNpmResolution(
-    val wasUpToDate: Boolean,
     val rootProject: Project,
     private val projects: Map<Project, KotlinProjectNpmResolution>
-) : KotlinNpmResolutionManager.ResolutionStateData {
+) {
     operator fun get(project: Project) = projects[project] ?: KotlinProjectNpmResolution.empty(project)
-
-    override val compilations: Collection<KotlinJsCompilation>
-        get() = projects.values.flatMap { it.npmProjects.map { it.npmProject.compilation } }
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/resolver/KotlinCompilationNpmResolver.kt

@@ -25,6 +25,9 @@ import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinPackageJsonTask
 import java.io.File
 import java.io.Serializable
 
+/**
+ * See [KotlinNpmResolutionManager] for details about resolution process.
+ */
 internal class KotlinCompilationNpmResolver(
     val projectResolver: KotlinProjectNpmResolver,
     val compilation: KotlinJsCompilation

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/resolver/KotlinProjectNpmResolver.kt

@@ -19,6 +19,9 @@ import org.jetbrains.kotlin.gradle.targets.js.npm.RequiresNpmDependencies
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinProjectNpmResolution
 import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinPackageJsonTask
 
+/**
+ * See [KotlinNpmResolutionManager] for details about resolution process.
+ */
 internal class KotlinProjectNpmResolver(
     val project: Project,
     val resolver: KotlinRootNpmResolver

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/resolver/KotlinRootNpmResolver.kt

@@ -15,17 +15,12 @@ import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinCompilationNpmR
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinRootNpmResolution
 
 /**
- * Generates `package.json` file for [NpmProject] with npm or js dependencies and
+ * See [KotlinNpmResolutionManager] for details about resolution process.
- * runs selected [NodeJsRootExtension.packageManager] to download and install all of it's dependencies.
- *
- * All [NpmDependency] for configurations related to kotlin/js will be added to `package.json`.
- * For external gradle modules, fake npm packages will be created and added to `package.json`
- * as path to directory.
  */
 internal class KotlinRootNpmResolver internal constructor(
     val nodeJs: NodeJsRootExtension,
     val forceFullResolve: Boolean
-) : KotlinNpmResolutionManager.ResolutionStateData {
+) {
     val rootProject: Project
         get() = nodeJs.rootProject
 
@@ -44,7 +39,7 @@ internal class KotlinRootNpmResolver internal constructor(
 
     operator fun get(project: Project) = projectResolvers[project] ?: error("$project is not configured for JS usage")
 
-    override val compilations: Collection<KotlinJsCompilation>
+    val compilations: Collection<KotlinJsCompilation>
         get() = projectResolvers.values.flatMap { it.compilationResolvers.map { it.compilation } }
 
     fun findDependentResolver(src: Project, target: Project): KotlinCompilationNpmResolver? {
@@ -67,7 +62,7 @@ internal class KotlinRootNpmResolver internal constructor(
     /**
      * Don't use directly, use [NodeJsRootExtension.resolveIfNeeded] instead.
      */
-    internal fun close(): KotlinRootNpmResolution {
+    internal fun close(skipPackageManager: Boolean): KotlinRootNpmResolution {
         check(!closed)
         closed = true
 
@@ -78,18 +73,31 @@ internal class KotlinRootNpmResolver internal constructor(
 
         gradleNodeModules.close()
 
-        val wasUpToDate = when {
+        fun result() = KotlinRootNpmResolution(rootProject, projectResolutions)
-            allNpmPackages.any { it.externalNpmDependencies.isNotEmpty() } -> {
-                nodeJs.packageManager.resolveRootProject(rootProject, allNpmPackages) == NpmApi.Result.upToDate
-            }
-            projectResolvers.values.any { it.taskRequirements.hasNodeModulesDependentTasks } -> {
-                NpmSimpleLinker(nodeJs).link(allNpmPackages)
-                true // todo
-            }
-            else -> true
-        }
 
-        return KotlinRootNpmResolution(wasUpToDate, rootProject, projectResolutions)
+        // we need manual up-to-date checking to avoid call package manager during
+        // idea import if nothing was changed
+        // we should call it even kotlinNpmInstall task is up-to-date (skipPackageManager is true)
+        // because our upToDateChecks saves state for next execution
+        val upToDateChecks = allNpmPackages.map {
+            PackageJsonUpToDateCheck(it.npmProject)
+        }
+        if (upToDateChecks.all { it.upToDate }) return result()
+
+        if (skipPackageManager) {
+            upToDateChecks.forEach { it.commit() }
+            return result()
+        } else {
+            if (allNpmPackages.any { it.externalNpmDependencies.isNotEmpty() }) {
+                nodeJs.packageManager.resolveRootProject(rootProject, allNpmPackages)
+            } else if (projectResolvers.values.any { it.taskRequirements.hasNodeModulesDependentTasks }) {
+                NpmSimpleLinker(nodeJs).link(allNpmPackages)
+            }
+
+            upToDateChecks.forEach { it.commit() }
+
+            return result()
+        }
     }
 
     private fun removeOutdatedPackages(nodeJs: NodeJsRootExtension, allNpmPackages: List<KotlinCompilationNpmResolution>) {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/tasks/KotlinNpmInstallTask.kt

@@ -19,11 +19,12 @@ open class KotlinNpmInstallTask : DefaultTask() {
     }
 
     private val nodeJs get() = NodeJsRootPlugin.apply(project.rootProject)
+    private val resolutionManager get() = nodeJs.npmResolutionManager
 
     @Suppress("unused")
     @get:InputFiles
     val packageJsonFiles: Collection<File>
-        get() = nodeJs.npmResolutionManager.resolutionState.compilations.map { it.npmProject.packageJsonFile }
+        get() = resolutionManager.packageJsonFiles
 
     @get:OutputFile
     val yarnLock: File
@@ -31,7 +32,7 @@ open class KotlinNpmInstallTask : DefaultTask() {
 
     @TaskAction
     fun resolve() {
-        nodeJs.npmResolutionManager.installIfNeeded(requireNotInstalled = true)
+        resolutionManager.install()
     }
 
     companion object {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnSimple.kt

@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.gradle.targets.js.yarn
 
 import org.gradle.api.Project
 import org.jetbrains.kotlin.gradle.targets.js.npm.NpmApi
+import org.jetbrains.kotlin.gradle.targets.js.npm.PackageJsonUpToDateCheck
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinCompilationNpmResolution
 
 object YarnSimple : YarnBasics() {
@@ -15,11 +16,11 @@ object YarnSimple : YarnBasics() {
 
         val project = resolvedNpmProject.project
 
-        YarnUpToDateCheck(resolvedNpmProject.npmProject).updateIfNeeded {
+        PackageJsonUpToDateCheck(resolvedNpmProject.npmProject).updateIfNeeded {
             yarnExec(project, resolvedNpmProject.npmProject.dir, NpmApi.resolveOperationDescription("yarn for ${project.path}"))
             yarnLockReadTransitiveDependencies(resolvedNpmProject.npmProject.dir, resolvedNpmProject.externalNpmDependencies)
         }
     }
 
-    override fun resolveRootProject(rootProject: Project, subProjects: Collection<KotlinCompilationNpmResolution>) = NpmApi.Result.upToDate
+    override fun resolveRootProject(rootProject: Project, subProjects: Collection<KotlinCompilationNpmResolution>) = Unit
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnWorkspaces.kt

@@ -14,7 +14,7 @@ import java.io.File
 object YarnWorkspaces : YarnBasics() {
     override fun resolveProject(resolvedNpmProject: KotlinCompilationNpmResolution) = Unit
 
-    override fun resolveRootProject(rootProject: Project, subProjects: Collection<KotlinCompilationNpmResolution>): NpmApi.Result {
+    override fun resolveRootProject(rootProject: Project, subProjects: Collection<KotlinCompilationNpmResolution>) {
         check(rootProject == rootProject.rootProject)
         setup(rootProject)
         return resolveWorkspaces(rootProject, subProjects)
@@ -23,25 +23,13 @@ object YarnWorkspaces : YarnBasics() {
     private fun resolveWorkspaces(
         rootProject: Project,
         npmProjects: Collection<KotlinCompilationNpmResolution>
-    ): NpmApi.Result {
+    ) {
-        val upToDateChecks = npmProjects.map {
-            YarnUpToDateCheck(it.npmProject)
-        }
-
-        if (upToDateChecks.all { it.upToDate }) return NpmApi.Result.upToDate
-
         val nodeJsWorldDir = NodeJsRootPlugin.apply(rootProject).rootPackageDir
 
         saveRootProjectWorkspacesPackageJson(rootProject, npmProjects, nodeJsWorldDir)
 
         yarnExec(rootProject, nodeJsWorldDir, NpmApi.resolveOperationDescription("yarn"))
         yarnLockReadTransitiveDependencies(nodeJsWorldDir, npmProjects.flatMap { it.externalNpmDependencies })
-
-        upToDateChecks.forEach {
-            it.commit()
-        }
-
-        return NpmApi.Result.executed
     }
 
     private fun saveRootProjectWorkspacesPackageJson(