Gradle, JS, NpmResolutionManager: cleanup and document states

2019-07-07 16:06:17 +03:00
parent 327d75dc5d
commit 3658ad785f
10 changed files with 100 additions and 117 deletions
@@ -5,6 +5,7 @@

 package org.jetbrains.kotlin.gradle.targets.js.npm

+import org.gradle.api.Incubating
 import org.gradle.api.Project
 import org.gradle.api.Task
 import org.jetbrains.kotlin.gradle.internal.isInIdeaSync
@@ -14,14 +15,14 @@ import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinCompilationNpmR
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinProjectNpmResolution
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinRootNpmResolution
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolver.KotlinRootNpmResolver
-import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinNpmInstallTask
+import java.io.File

 /**
 * # NPM resolution state manager
 *
 * ## Resolving process from Gradle
 *
- * **configuring**. Initial state. [NpmResolverPlugin] should be applied for each project
+ * **configuring**. Global initial state. [NpmResolverPlugin] should be applied for each project
 * that requires NPM resolution. This plugin should be applied only after kotlin plugin.
 * When applied, [KotlinProjectNpmResolver] will be created for the corresponding project
 * and will subscribe to all js compilations.
@@ -43,67 +44,67 @@ import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinNpmInstallTask
 *
 * Note that package.json will be executed only for required compilations, while other may be missed.
 *
- * **installing**
- *
- * **installed**
- *
- *
- *
- * - resolving:
- *   - [KotlinPackageJsonTask] task executed for each affected compilation:
- *      - ensure that all needed package.json files created
- *   - global [KotlinNpmInstallTask] executed:
- *      - call package manager to sync node_modules
- *
- * Resolved state requested by calling [requireAlreadyInstalled] from this places:
- *  - tasks that requires node_modules
- *  - before calling [NpmProject.require]
- *
- * Also resolved state used when getting files of [NpmDependency]. In this case
- * resolved project is not required. [NpmDependency] will not return any files if
- * ... [requireAlreadyResolvedOrNullIfResolvingNow]
- *
- * Note that [KotlinPackageJsonTask] executed for required compilations only.
- * [KotlinNpmInstallTask] may not be executed if none of package.json tasks executed.
- * In this case [_resolutionState] will remain unclosed and will be closed
- * on first [requireAlreadyInstalled] call with check that everything is already up-to-date.
+ * **installed**. Global final state. Initiated by executing global `kotlinNpmInstall` task.
+ * All created package.json files will be gathered and package manager will be executed.
+ * Package manager will create lock file, that will be parsed for transitive npm dependencies
+ * that will be added to the root [NpmDependency] objects. `kotlinNpmInstall` task may be up-to-date.
+ * In this case, installed state will be reached by first call of [installIfNeeded] without exeucting
+ * package manager.
 *
+ * Resolution will be used from [NpmDependency] by calling [getNpmDependencyResolvedCompilation].
+ * Also resolution will be checked before calling [NpmProject.require] and executing any task
+ * that requires npm dependencies or node_modules.
 *
+ * Also user can call [requireInstalled] to get resolution info.
 *
 * ## Resolving process during Idea import
+ *
+ * In this case [forceFullResolve] will be set, and all compilations will be resolved
+ * even without `packageJson` task execution.
+ *
+ * During building gradle project model, all [NpmDependency] will be asked for there files,
+ * and [getNpmDependencyResolvedCompilation] will be called. In the [forceFullResolve] mode
+ * project will be fully resolved: all `package.json` files will be created, and package manager
+ * will be called. We are manually tracking package.json files contents to avoid calling package manager
+ * is nothing was changes.
+ *
+ * Note that in this case resolution process will be performed outside of task execution.
 */
 class KotlinNpmResolutionManager(val nodeJsSettings: NodeJsRootExtension) {
-    internal val resolutionState: ResolutionState
-        get() = _resolutionState
-
    private val forceFullResolve: Boolean
        get() = isInIdeaSync

    @Volatile
-    private var _resolutionState: ResolutionState =
+    private var state: ResolutionState =
        ResolutionState.Configuring(
            KotlinRootNpmResolver(nodeJsSettings, forceFullResolve)
        )

-    internal sealed class ResolutionState : ResolutionStateData {
-        class Configuring(val resolver: KotlinRootNpmResolver) : ResolutionState(), ResolutionStateData by resolver
-        class Installing(val resolver: KotlinRootNpmResolver) : ResolutionState(), ResolutionStateData by resolver
-        class Installed(val resolved: KotlinRootNpmResolution) : ResolutionState(), ResolutionStateData by resolved
+    internal sealed class ResolutionState {
+        class Configuring(val resolver: KotlinRootNpmResolver) : ResolutionState()
+        class Installed(val resolved: KotlinRootNpmResolution) : ResolutionState()
    }

-    interface ResolutionStateData {
-        val compilations: Collection<KotlinJsCompilation>
-    }
+    @Incubating
+    fun requireInstalled() = (state as ResolutionState.Installed).resolved

    internal fun requireConfiguringState(): KotlinRootNpmResolver =
-        (resolutionState as? ResolutionState.Configuring ?: error("NPM Dependencies already resolved and installed")).resolver
+        (this.state as? ResolutionState.Configuring ?: error("NPM Dependencies already resolved and installed")).resolver
+
+    internal fun install() = installIfNeeded(requireNotInstalled = true)
+
+    internal fun requireAlreadyInstalled(project: Project, reason: String = ""): KotlinProjectNpmResolution =
+        installIfNeeded(requireUpToDateReason = reason)[project]
+
+    internal val packageJsonFiles: Collection<File>
+        get() = (state as ResolutionState.Configuring).resolver.compilations.map { it.npmProject.packageJsonFile }

    /**
     * @param requireUpToDateReason Check that project already resolved,
     * or it is up-to-date but just not closed. Show given message if it is not.
     * @param requireNotInstalled Check that project is not resolved
     */
-    internal fun installIfNeeded(
+    private fun installIfNeeded(
        requireUpToDateReason: String? = null,
        requireNotInstalled: Boolean = false
    ): KotlinRootNpmResolution {
@@ -112,29 +113,23 @@ class KotlinNpmResolutionManager(val nodeJsSettings: NodeJsRootExtension) {
            return resolution
        }

-        val state0 = _resolutionState
+        val state0 = this.state
        val resolution = when (state0) {
            is ResolutionState.Installed -> alreadyResolved(state0.resolved)
-            is ResolutionState.Installing,
            is ResolutionState.Configuring -> {
                synchronized(this) {
-                    val state1 = _resolutionState
+                    val state1 = this.state
                    when (state1) {
                        is ResolutionState.Installed -> alreadyResolved(state1.resolved)
-                        is ResolutionState.Installing -> error("Resolution state sync failed")
                        is ResolutionState.Configuring -> {
-                            val state2 =
-                                ResolutionState.Installing(state1.resolver)
-                            _resolutionState = state2
-                            state1.resolver.close().also {
-                                check(_resolutionState == state2) {
-                                    "Resolution state sync failed: $_resolutionState != $state2"
-                                }
-                                _resolutionState =
-                                    ResolutionState.Installed(it)
-                                if (requireUpToDateReason != null && !it.wasUpToDate) {
-                                    error("NPM dependencies should be resolved $requireUpToDateReason")
-                                }
+                            val upToDate = nodeJsSettings.npmInstallTask.state.upToDate
+                            if (requireUpToDateReason != null && !upToDate) {
+                                error("NPM dependencies should be resolved $requireUpToDateReason")
+                            }
+
+                            val skipPackageManager = upToDate && !forceFullResolve
+                            state1.resolver.close(skipPackageManager).also {
+                                this.state = ResolutionState.Installed(it)
                            }
                        }
                    }
@@ -145,9 +140,6 @@ class KotlinNpmResolutionManager(val nodeJsSettings: NodeJsRootExtension) {
        return resolution
    }

-    internal fun requireAlreadyInstalled(project: Project, reason: String = ""): KotlinProjectNpmResolution =
-        installIfNeeded(requireUpToDateReason = reason)[project]
-
    internal fun getNpmDependencyResolvedCompilation(npmDependency: NpmDependency): KotlinCompilationNpmResolution {
        val project = npmDependency.project

@@ -158,18 +150,14 @@ class KotlinNpmResolutionManager(val nodeJsSettings: NodeJsRootExtension) {
                // may return null only during npm resolution
                // (it can be called since NpmDependency added to configuration that
                // requires resolve to build package.json, in this case we should just skip this call)
-                val state0 = resolutionState
+                val state0 = state
                when (state0) {
                    is ResolutionState.Installed -> state0.resolved[project]
-                    is ResolutionState.Installing -> null
                    is ResolutionState.Configuring -> error("Cannot use NpmDependency before :kotlinNpmInstall task execution")
                }
            }

-        return when (resolvedProject) {
-            null -> null
-            else -> resolvedProject.npmProjectsByNpmDependency[npmDependency] ?: error("NPM project resolved without $this")
-        }
+        return resolvedProject.npmProjectsByNpmDependency[npmDependency] ?: error("NPM project resolved without $this")
    }

    internal fun <T> checkRequiredDependencies(task: T)
@@ -16,14 +16,10 @@ interface NpmApi {

    fun resolveProject(resolvedNpmProject: KotlinCompilationNpmResolution)

-    enum class Result {
-        upToDate, executed
-    }
-
    fun resolveRootProject(
        rootProject: Project,
        subProjects: Collection<KotlinCompilationNpmResolution>
-    ): Result
+    )

    companion object {
        fun resolveOperationDescription(packageManagerTitle: String): String =
@@ -1,19 +1,18 @@
 /*
- * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
- * that can be found in the license/LICENSE.txt file.
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
 */

-package org.jetbrains.kotlin.gradle.targets.js.yarn
+package org.jetbrains.kotlin.gradle.targets.js.npm

 import org.gradle.api.Project
 import org.gradle.api.internal.project.ProjectInternal
 import org.gradle.internal.hash.FileHasher
 import org.jetbrains.kotlin.daemon.common.toHexString
 import org.jetbrains.kotlin.gradle.internal.ensureParentDirsCreated
-import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProject
 import java.io.File

-class YarnUpToDateCheck(val npmProject: NpmProject) {
+class PackageJsonUpToDateCheck(val npmProject: NpmProject) {
    val project: Project
        get() = npmProject.project

@@ -10,12 +10,8 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinJsCompilation
 import org.jetbrains.kotlin.gradle.targets.js.npm.KotlinNpmResolutionManager

 class KotlinRootNpmResolution(
-    val wasUpToDate: Boolean,
    val rootProject: Project,
    private val projects: Map<Project, KotlinProjectNpmResolution>
-) : KotlinNpmResolutionManager.ResolutionStateData {
+) {
    operator fun get(project: Project) = projects[project] ?: KotlinProjectNpmResolution.empty(project)
-
-    override val compilations: Collection<KotlinJsCompilation>
-        get() = projects.values.flatMap { it.npmProjects.map { it.npmProject.compilation } }
 }
@@ -25,6 +25,9 @@ import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinPackageJsonTask
 import java.io.File
 import java.io.Serializable

+/**
+ * See [KotlinNpmResolutionManager] for details about resolution process.
+ */
 internal class KotlinCompilationNpmResolver(
    val projectResolver: KotlinProjectNpmResolver,
    val compilation: KotlinJsCompilation
@@ -19,6 +19,9 @@ import org.jetbrains.kotlin.gradle.targets.js.npm.RequiresNpmDependencies
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinProjectNpmResolution
 import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinPackageJsonTask

+/**
+ * See [KotlinNpmResolutionManager] for details about resolution process.
+ */
 internal class KotlinProjectNpmResolver(
    val project: Project,
    val resolver: KotlinRootNpmResolver
@@ -15,17 +15,12 @@ import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinCompilationNpmR
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinRootNpmResolution

 /**
- * Generates `package.json` file for [NpmProject] with npm or js dependencies and
- * runs selected [NodeJsRootExtension.packageManager] to download and install all of it's dependencies.
- *
- * All [NpmDependency] for configurations related to kotlin/js will be added to `package.json`.
- * For external gradle modules, fake npm packages will be created and added to `package.json`
- * as path to directory.
+ * See [KotlinNpmResolutionManager] for details about resolution process.
 */
 internal class KotlinRootNpmResolver internal constructor(
    val nodeJs: NodeJsRootExtension,
    val forceFullResolve: Boolean
-) : KotlinNpmResolutionManager.ResolutionStateData {
+) {
    val rootProject: Project
        get() = nodeJs.rootProject

@@ -44,7 +39,7 @@ internal class KotlinRootNpmResolver internal constructor(

    operator fun get(project: Project) = projectResolvers[project] ?: error("$project is not configured for JS usage")

-    override val compilations: Collection<KotlinJsCompilation>
+    val compilations: Collection<KotlinJsCompilation>
        get() = projectResolvers.values.flatMap { it.compilationResolvers.map { it.compilation } }

    fun findDependentResolver(src: Project, target: Project): KotlinCompilationNpmResolver? {
@@ -67,7 +62,7 @@ internal class KotlinRootNpmResolver internal constructor(
    /**
     * Don't use directly, use [NodeJsRootExtension.resolveIfNeeded] instead.
     */
-    internal fun close(): KotlinRootNpmResolution {
+    internal fun close(skipPackageManager: Boolean): KotlinRootNpmResolution {
        check(!closed)
        closed = true

@@ -78,18 +73,31 @@ internal class KotlinRootNpmResolver internal constructor(

        gradleNodeModules.close()

-        val wasUpToDate = when {
-            allNpmPackages.any { it.externalNpmDependencies.isNotEmpty() } -> {
-                nodeJs.packageManager.resolveRootProject(rootProject, allNpmPackages) == NpmApi.Result.upToDate
-            }
-            projectResolvers.values.any { it.taskRequirements.hasNodeModulesDependentTasks } -> {
-                NpmSimpleLinker(nodeJs).link(allNpmPackages)
-                true // todo
-            }
-            else -> true
-        }
+        fun result() = KotlinRootNpmResolution(rootProject, projectResolutions)

-        return KotlinRootNpmResolution(wasUpToDate, rootProject, projectResolutions)
+        // we need manual up-to-date checking to avoid call package manager during
+        // idea import if nothing was changed
+        // we should call it even kotlinNpmInstall task is up-to-date (skipPackageManager is true)
+        // because our upToDateChecks saves state for next execution
+        val upToDateChecks = allNpmPackages.map {
+            PackageJsonUpToDateCheck(it.npmProject)
+        }
+        if (upToDateChecks.all { it.upToDate }) return result()
+
+        if (skipPackageManager) {
+            upToDateChecks.forEach { it.commit() }
+            return result()
+        } else {
+            if (allNpmPackages.any { it.externalNpmDependencies.isNotEmpty() }) {
+                nodeJs.packageManager.resolveRootProject(rootProject, allNpmPackages)
+            } else if (projectResolvers.values.any { it.taskRequirements.hasNodeModulesDependentTasks }) {
+                NpmSimpleLinker(nodeJs).link(allNpmPackages)
+            }
+
+            upToDateChecks.forEach { it.commit() }
+
+            return result()
+        }
    }

    private fun removeOutdatedPackages(nodeJs: NodeJsRootExtension, allNpmPackages: List<KotlinCompilationNpmResolution>) {
@@ -19,11 +19,12 @@ open class KotlinNpmInstallTask : DefaultTask() {
    }

    private val nodeJs get() = NodeJsRootPlugin.apply(project.rootProject)
+    private val resolutionManager get() = nodeJs.npmResolutionManager

    @Suppress("unused")
    @get:InputFiles
    val packageJsonFiles: Collection<File>
-        get() = nodeJs.npmResolutionManager.resolutionState.compilations.map { it.npmProject.packageJsonFile }
+        get() = resolutionManager.packageJsonFiles

    @get:OutputFile
    val yarnLock: File
@@ -31,7 +32,7 @@ open class KotlinNpmInstallTask : DefaultTask() {

    @TaskAction
    fun resolve() {
-        nodeJs.npmResolutionManager.installIfNeeded(requireNotInstalled = true)
+        resolutionManager.install()
    }

    companion object {
@@ -7,6 +7,7 @@ package org.jetbrains.kotlin.gradle.targets.js.yarn

 import org.gradle.api.Project
 import org.jetbrains.kotlin.gradle.targets.js.npm.NpmApi
+import org.jetbrains.kotlin.gradle.targets.js.npm.PackageJsonUpToDateCheck
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolved.KotlinCompilationNpmResolution

 object YarnSimple : YarnBasics() {
@@ -15,11 +16,11 @@ object YarnSimple : YarnBasics() {

        val project = resolvedNpmProject.project

-        YarnUpToDateCheck(resolvedNpmProject.npmProject).updateIfNeeded {
+        PackageJsonUpToDateCheck(resolvedNpmProject.npmProject).updateIfNeeded {
            yarnExec(project, resolvedNpmProject.npmProject.dir, NpmApi.resolveOperationDescription("yarn for ${project.path}"))
            yarnLockReadTransitiveDependencies(resolvedNpmProject.npmProject.dir, resolvedNpmProject.externalNpmDependencies)
        }
    }

-    override fun resolveRootProject(rootProject: Project, subProjects: Collection<KotlinCompilationNpmResolution>) = NpmApi.Result.upToDate
+    override fun resolveRootProject(rootProject: Project, subProjects: Collection<KotlinCompilationNpmResolution>) = Unit
 }
@@ -14,7 +14,7 @@ import java.io.File
 object YarnWorkspaces : YarnBasics() {
    override fun resolveProject(resolvedNpmProject: KotlinCompilationNpmResolution) = Unit

-    override fun resolveRootProject(rootProject: Project, subProjects: Collection<KotlinCompilationNpmResolution>): NpmApi.Result {
+    override fun resolveRootProject(rootProject: Project, subProjects: Collection<KotlinCompilationNpmResolution>) {
        check(rootProject == rootProject.rootProject)
        setup(rootProject)
        return resolveWorkspaces(rootProject, subProjects)
@@ -23,25 +23,13 @@ object YarnWorkspaces : YarnBasics() {
    private fun resolveWorkspaces(
        rootProject: Project,
        npmProjects: Collection<KotlinCompilationNpmResolution>
-    ): NpmApi.Result {
-        val upToDateChecks = npmProjects.map {
-            YarnUpToDateCheck(it.npmProject)
-        }
-
-        if (upToDateChecks.all { it.upToDate }) return NpmApi.Result.upToDate
-
+    ) {
        val nodeJsWorldDir = NodeJsRootPlugin.apply(rootProject).rootPackageDir

        saveRootProjectWorkspacesPackageJson(rootProject, npmProjects, nodeJsWorldDir)

        yarnExec(rootProject, nodeJsWorldDir, NpmApi.resolveOperationDescription("yarn"))
        yarnLockReadTransitiveDependencies(nodeJsWorldDir, npmProjects.flatMap { it.externalNpmDependencies })
-
-        upToDateChecks.forEach {
-            it.commit()
-        }
-
-        return NpmApi.Result.executed
    }

    private fun saveRootProjectWorkspacesPackageJson(