Gradle, yarn: get recursively resolved dependencies

#KT-31018 Fixed #KT-31016 Fixed #KT-31017 Fixed
2019-04-13 14:41:26 +03:00
parent b96f26cc77
commit 4f4fc74b16
10 changed files with 164 additions and 41 deletions
@@ -35,6 +35,9 @@ internal fun Project.createKotlinExtension(extensionClass: KClass<out KotlinProj
    return kotlinExtension
 }

+internal val Project.kotlinExtensionOrNull: KotlinProjectExtension?
+    get() = extensions.findByName(KOTLIN_PROJECT_EXTENSION_NAME) as KotlinProjectExtension?
+
 internal val Project.kotlinExtension: KotlinProjectExtension
    get() = extensions.getByName(KOTLIN_PROJECT_EXTENSION_NAME) as KotlinProjectExtension

@@ -30,6 +30,7 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinMultiplatformPlugin
 import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinUsages
 import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSetFactory
 import org.jetbrains.kotlin.gradle.targets.js.KotlinJsPlugin
+import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin
 import org.jetbrains.kotlin.gradle.tasks.KOTLIN_COMPILER_EMBEDDABLE
 import org.jetbrains.kotlin.gradle.tasks.KOTLIN_MODULE_GROUP
 import org.jetbrains.kotlin.gradle.utils.checkGradleCompatibility
@@ -27,7 +27,7 @@ class KotlinJsTargetConfigurator(kotlinPluginVersion: String) :

        platformTarget.compilations.all {
            it.compileKotlinTask.kotlinOptions {
-                moduleKind = "umd"
+                moduleKind = "commonjs"
                sourceMap = true
                sourceMapEmbedSources = null
            }
@@ -19,13 +19,18 @@ interface NpmApi {
        subprojects: MutableList<NpmResolver.NpmPackage>
    )

+    /**
+     * Change contents of root's project `package.json`.
+     * @return true if `package.json` is required even it is empty
+     */
    @Suppress("EXPOSED_PARAMETER_TYPE")
-    fun hookRootPackage(rootProject: Project, rootPackageJson: PackageJson, allWorkspaces: Collection<NpmResolver.NpmPackage>) {
+    fun hookRootPackage(
+        rootProject: Project,
+        rootPackageJson: PackageJson,
+        allWorkspaces: Collection<NpmResolver.NpmPackage>
+    ): Boolean = false

-    }
-
-    open val hoistGradleNodeModules
-        get() = false
+    open fun getHoistGradleNodeModules(project: Project) = false

    companion object {
        fun resolveOperationDescription(packageManagerTitle: String): String =
@@ -10,51 +10,74 @@ import org.gradle.api.artifacts.Dependency
 import org.gradle.api.artifacts.FileCollectionDependency
 import org.gradle.api.artifacts.SelfResolvingDependency
 import org.gradle.api.file.FileCollection
+import org.gradle.api.internal.artifacts.DependencyResolveContext
+import org.gradle.api.internal.artifacts.ResolvableDependency
 import org.gradle.api.internal.artifacts.dependencies.SelfResolvingDependencyInternal
 import org.gradle.api.tasks.TaskDependency
-import java.io.File
 import org.gradle.internal.component.local.model.DefaultLibraryBinaryIdentifier
 import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver.ResolutionCallResult.*
+import java.io.File

 data class NpmDependency(
-    private val project: Project,
+    internal val project: Project,
    private val org: String?,
    private val name: String,
    private val version: String
 ) : SelfResolvingDependency,
    SelfResolvingDependencyInternal,
+    ResolvableDependency,
    FileCollectionDependency {
    override fun getGroup(): String? = org

+    internal val dependencies = mutableSetOf<NpmDependency>()
+
    override fun resolve(): MutableSet<File> {
+        resolveProject() ?: return mutableSetOf()
+
+        val all = mutableSetOf<File>()
+        val visited = mutableSetOf<NpmDependency>()
+
+        fun visit(item: NpmDependency) {
+            if (item in visited) return
+            visited.add(item)
+            item.tryFindNodeModule()?.let {
+                all.add(it)
+            }
+        }
+
+        visit(this)
+
+        return all
+    }
+
+    internal fun resolveProject(): NpmResolver.ResolvedProject? {
        val result = NpmResolver.resolve(project)

        return when (result) {
-            is AlreadyInProgress -> mutableSetOf()
+            is AlreadyInProgress -> null
            is AlreadyResolved -> {
                check(this in result.resolution.dependencies) {
                    "Project hierarchy is already resolved in NPM without $this"
                }

-                tryFindNodeModule()
+                result.resolution
            }
            is ResolvedNow -> {
                check(this in result.resolution.dependencies) {
                    "$this was not visited during resolution"
                }

-                tryFindNodeModule()
+                result.resolution
            }
        }
    }

-    private fun tryFindNodeModule(): MutableSet<File> {
-        // todo: transitive dependencies
+    private fun tryFindNodeModule(): File? {
        var p = project
        do {
-            val result = p.rootDir.resolve("node_modules/$key")
-            if (result.exists()) return mutableSetOf(result)
-            p = project.parent ?: return mutableSetOf()
+            val result = NpmProjectLayout[p].nodeModulesDir.resolve(key)
+            if (result.exists()) return result
+            p = project.parent ?: return null
        } while (true)
    }

@@ -62,6 +85,13 @@ data class NpmDependency(

    override fun toString() = "$key: $version"

+    override fun resolve(context: DependencyResolveContext) {
+        resolve()
+        dependencies.forEach {
+            context.add(it)
+        }
+    }
+
    override fun resolve(transitive: Boolean): MutableSet<File> = resolve()

    override fun getFiles(): FileCollection = project.files(resolve())
@@ -54,10 +54,9 @@ internal class NpmResolver private constructor(val rootProject: Project) {
        class ResolvedNow(val resolution: ResolvedProject) : ResolutionCallResult()
    }

-    private val nodeJs = NodeJsPlugin.apply(rootProject).root
-    private val npmProject = rootProject.npmProject
+    private val nodeJs = NodeJsPlugin.apply(rootProject)
    private val packageManager = nodeJs.packageManager
-    private val hoistGradleNodeModules = npmProject.hoistGradleNodeModules
+    private val hoistGradleNodeModules = packageManager.getHoistGradleNodeModules(rootProject)
    private val npmPackages = mutableListOf<NpmPackage>()
    private val gson = GsonBuilder()
        .setPrettyPrinting()
@@ -10,11 +10,15 @@ class PackageJson(
    var version: String
 ) {
    val empty: Boolean
-        get() = private == null && workspaces == null && dependencies.isEmpty()
+        get() = private == null &&
+                workspaces == null &&
+                dependencies.isEmpty() &&
+                devDependencies.isEmpty()

    var private: Boolean? = null

    var workspaces: Collection<String>? = null

+    val devDependencies = mutableMapOf<String, String>()
    val dependencies = mutableMapOf<String, String>()
 }
@@ -6,55 +6,119 @@
 package org.jetbrains.kotlin.gradle.targets.js.yarn

 import org.gradle.api.Project
+import org.gradle.api.internal.project.ProjectInternal
+import org.gradle.internal.hash.FileHasher
+import org.jetbrains.kotlin.daemon.common.toHexString
 import org.jetbrains.kotlin.gradle.internal.execWithProgress
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin
-import org.jetbrains.kotlin.gradle.targets.js.npm.NpmApi
-import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProjectLayout
-import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver
-import org.jetbrains.kotlin.gradle.targets.js.npm.PackageJson
-import org.slf4j.LoggerFactory
+import org.jetbrains.kotlin.gradle.targets.js.npm.*
+import java.io.File

 object Yarn : NpmApi {
-    private val log = LoggerFactory.getLogger("org.jetbrains.kotlin.gradle.targets.js.yarn.Yarn")
-
    override fun setup(project: Project) {
-        YarnPlugin[project].executeSetup()
+        YarnPlugin.apply(project).executeSetup()
    }

+    val Project.packageJsonHashFile: File
+        get() = buildDir.resolve("package.json.hash")
+
    fun yarnExec(
        project: Project,
        description: String,
        vararg args: String,
        npmProjectLayout: NpmProjectLayout = NpmProjectLayout[project]
    ) {
-        val nodeJsEnv = NodeJsPlugin[project].buildEnv()
-        val yarnEnv = YarnPlugin[project].buildEnv()
+        val nodeJsEnv = NodeJsPlugin.apply(project).environment
+        val yarnEnv = YarnPlugin.apply(project).environment
+
+        val packageJsonHashFile = project.packageJsonHashFile
+        val packageJsonHash = if (packageJsonHashFile.exists()) packageJsonHashFile.readText() else null
+
+        val hasher = (project as ProjectInternal).services.get(FileHasher::class.java)
+        val hash = hasher.hash(npmProjectLayout.packageJsonFile).toByteArray().toHexString()
+
+        if (packageJsonHash == hash) return
+
+        packageJsonHashFile.writeText(hash)
+
+        val nodeWorkDir = npmProjectLayout.nodeWorkDir

        project.execWithProgress(description) { exec ->
            exec.executable = nodeJsEnv.nodeExecutable
            exec.args = listOf(yarnEnv.home.resolve("bin/yarn.js").absolutePath) + args
-            exec.workingDir = npmProjectLayout.nodeWorkDir
+            exec.workingDir = nodeWorkDir
+        }
+    }
+
+    private fun yarnLockReadTransitiveDependencies(
+        nodeWorkDir: File,
+        srcDependenciesList: Collection<NpmDependency>
+    ) {
+        val yarnLock = nodeWorkDir.resolve("yarn.lock")
+        if (yarnLock.isFile) {
+            val byKey = YarnLock.parse(yarnLock).entries.associateBy { it.key }
+            val visited = mutableSetOf<NpmDependency>()
+
+            fun resolveRecursively(src: NpmDependency): NpmDependency {
+                if (src in visited) return src
+                visited.add(src)
+
+                val key = YarnLock.key(src.key, src.version)
+                val deps = byKey[key]
+                if (deps != null) {
+                    src.dependencies.addAll(deps.dependencies.map { dep ->
+                        resolveRecursively(
+                            NpmDependency(
+                                src.project,
+                                dep.group,
+                                dep.packageName,
+                                dep.version ?: "*"
+                            )
+                        )
+                    })
+                } else {
+                    // todo: [WARN] cannot find $key in yarn.lock
+                }
+
+                return src
+            }
+
+            srcDependenciesList.forEach { src ->
+                resolveRecursively(src)
+            }
        }
    }

    @Suppress("EXPOSED_PARAMETER_TYPE")
    override fun resolveProject(npmPackage: NpmResolver.NpmPackage) {
        val project = npmPackage.project
-        if (!YarnPlugin[project].checkUseWorkspace()) {
+        if (!project.yarn.useWorkspaces) {
            yarnExec(project, NpmApi.resolveOperationDescription("yarn for ${project.path}"))
+            yarnLockReadTransitiveDependencies(NpmProjectLayout[project].nodeWorkDir, npmPackage.npmDependencies)
        }
    }

    @Suppress("EXPOSED_PARAMETER_TYPE")
-    override fun hookRootPackage(rootProject: Project, rootPackageJson: PackageJson, allWorkspaces: Collection<NpmResolver.NpmPackage>) {
-        if (YarnPlugin[rootProject].checkUseWorkspace()) {
+    override fun hookRootPackage(
+        rootProject: Project,
+        rootPackageJson: PackageJson,
+        allWorkspaces: Collection<NpmResolver.NpmPackage>
+    ): Boolean {
+        if (rootProject.yarn.useWorkspaces) {
            rootPackageJson.private = true
            rootPackageJson.workspaces = allWorkspaces
                .filter { it.project != rootProject }
-                .map { it.project.rootDir.relativeTo(rootProject.rootDir).path }
+                .map { it.project.projectDir.relativeTo(rootProject.rootDir).path }
+
+            return true
        }
+
+        return false
    }

+    override fun getHoistGradleNodeModules(project: Project): Boolean =
+        project.rootProject.yarn.useWorkspaces
+
    @Suppress("EXPOSED_PARAMETER_TYPE")
    override fun resolveRootProject(
        rootProject: Project,
@@ -62,8 +126,9 @@ object Yarn : NpmApi {
    ) {
        check(rootProject == rootProject.rootProject)

-        if (YarnPlugin[rootProject].checkUseWorkspace()) {
+        if (rootProject.yarn.useWorkspaces) {
            yarnExec(rootProject, NpmApi.resolveOperationDescription("yarn"))
+            yarnLockReadTransitiveDependencies(NpmProjectLayout[rootProject].nodeWorkDir, subprojects.flatMap { it.npmDependencies })
        } else {
            if (subprojects.any { it.project != rootProject }) {
                // todo: proofread message
@@ -72,11 +137,15 @@ object Yarn : NpmApi {
                            "It is recommended to enable yarn workspaces to store common NPM dependencies in root project. " +
                            "To enable it add this to your root project: \n" +
                            "nodeJs { manageNodeModules = true } \n" +
-                            "yarn { useWorkspaces = true } \n" +
                            "Note: with `manageNodeModules` enabled, your `node_modules` and `package.json` files will be managed by " +
                            "Gradle, will be overridden during build and should be ignored in VCS."
                )
            }
        }
    }
+
+    override fun cleanProject(project: Project) {
+        super.cleanProject(project)
+        project.packageJsonHashFile.delete()
+    }
 }
@@ -6,7 +6,6 @@
 package org.jetbrains.kotlin.gradle.targets.js.yarn

 import com.google.gson.stream.JsonReader
-import org.jetbrains.kotlin.gradle.targets.js.npm.PackageJson
 import java.io.File
 import java.io.StringReader

@@ -17,11 +16,23 @@ data class YarnLock(val entries: List<Entry>) {
        val resolved: String?,
        val integrity: String?,
        val dependencies: List<Dependency>
-    )
+    ) {
+        val packageKey = key.substringBeforeLast("@")
+    }

    data class Dependency(val key: String, val version: String?) {
-        val scopedName: PackageJson.ScopedName
-            get() = PackageJson.scopedName(key)
+        val group: String?
+        val packageName: String
+
+        init {
+            if (key.contains("/")) {
+                group = key.substringBeforeLast("/").removePrefix("@")
+                packageName = key.substringAfterLast("/")
+            } else {
+                group = null
+                packageName = key
+            }
+        }
    }

    companion object {
@@ -0,0 +1 @@
+implementation-class=org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin
				`@@ -0,0 +1 @@`
				`implementation-class=org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin`