Gradle, yarn: get recursively resolved dependencies

#KT-31018 Fixed
	#KT-31016 Fixed
	#KT-31017 Fixed

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/dsl/KotlinProjectExtension.kt

@@ -35,6 +35,9 @@ internal fun Project.createKotlinExtension(extensionClass: KClass<out KotlinProj
     return kotlinExtension
 }
 
+internal val Project.kotlinExtensionOrNull: KotlinProjectExtension?
+    get() = extensions.findByName(KOTLIN_PROJECT_EXTENSION_NAME) as KotlinProjectExtension?
+
 internal val Project.kotlinExtension: KotlinProjectExtension
     get() = extensions.getByName(KOTLIN_PROJECT_EXTENSION_NAME) as KotlinProjectExtension
 

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt

@@ -30,6 +30,7 @@ import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinMultiplatformPlugin
 import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinUsages
 import org.jetbrains.kotlin.gradle.plugin.sources.DefaultKotlinSourceSetFactory
 import org.jetbrains.kotlin.gradle.targets.js.KotlinJsPlugin
+import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin
 import org.jetbrains.kotlin.gradle.tasks.KOTLIN_COMPILER_EMBEDDABLE
 import org.jetbrains.kotlin.gradle.tasks.KOTLIN_MODULE_GROUP
 import org.jetbrains.kotlin.gradle.utils.checkGradleCompatibility

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/KotlinJsTargetConfigurator.kt

@@ -27,7 +27,7 @@ class KotlinJsTargetConfigurator(kotlinPluginVersion: String) :
 
         platformTarget.compilations.all {
             it.compileKotlinTask.kotlinOptions {
-                moduleKind = "umd"
+                moduleKind = "commonjs"
                 sourceMap = true
                 sourceMapEmbedSources = null
             }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmApi.kt

@@ -19,13 +19,18 @@ interface NpmApi {
         subprojects: MutableList<NpmResolver.NpmPackage>
     )
 
+    /**
+     * Change contents of root's project `package.json`.
+     * @return true if `package.json` is required even it is empty
+     */
     @Suppress("EXPOSED_PARAMETER_TYPE")
-    fun hookRootPackage(rootProject: Project, rootPackageJson: PackageJson, allWorkspaces: Collection<NpmResolver.NpmPackage>) {
+    fun hookRootPackage(
+        rootProject: Project,
+        rootPackageJson: PackageJson,
+        allWorkspaces: Collection<NpmResolver.NpmPackage>
+    ): Boolean = false
 
-    }
+    open fun getHoistGradleNodeModules(project: Project) = false
-
-    open val hoistGradleNodeModules
-        get() = false
 
     companion object {
         fun resolveOperationDescription(packageManagerTitle: String): String =

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmDependency.kt

@@ -10,51 +10,74 @@ import org.gradle.api.artifacts.Dependency
 import org.gradle.api.artifacts.FileCollectionDependency
 import org.gradle.api.artifacts.SelfResolvingDependency
 import org.gradle.api.file.FileCollection
+import org.gradle.api.internal.artifacts.DependencyResolveContext
+import org.gradle.api.internal.artifacts.ResolvableDependency
 import org.gradle.api.internal.artifacts.dependencies.SelfResolvingDependencyInternal
 import org.gradle.api.tasks.TaskDependency
-import java.io.File
 import org.gradle.internal.component.local.model.DefaultLibraryBinaryIdentifier
 import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver.ResolutionCallResult.*
+import java.io.File
 
 data class NpmDependency(
-    private val project: Project,
+    internal val project: Project,
     private val org: String?,
     private val name: String,
     private val version: String
 ) : SelfResolvingDependency,
     SelfResolvingDependencyInternal,
+    ResolvableDependency,
     FileCollectionDependency {
     override fun getGroup(): String? = org
 
+    internal val dependencies = mutableSetOf<NpmDependency>()
+
     override fun resolve(): MutableSet<File> {
+        resolveProject() ?: return mutableSetOf()
+
+        val all = mutableSetOf<File>()
+        val visited = mutableSetOf<NpmDependency>()
+
+        fun visit(item: NpmDependency) {
+            if (item in visited) return
+            visited.add(item)
+            item.tryFindNodeModule()?.let {
+                all.add(it)
+            }
+        }
+
+        visit(this)
+
+        return all
+    }
+
+    internal fun resolveProject(): NpmResolver.ResolvedProject? {
         val result = NpmResolver.resolve(project)
 
         return when (result) {
-            is AlreadyInProgress -> mutableSetOf()
+            is AlreadyInProgress -> null
             is AlreadyResolved -> {
                 check(this in result.resolution.dependencies) {
                     "Project hierarchy is already resolved in NPM without $this"
                 }
 
-                tryFindNodeModule()
+                result.resolution
             }
             is ResolvedNow -> {
                 check(this in result.resolution.dependencies) {
                     "$this was not visited during resolution"
                 }
 
-                tryFindNodeModule()
+                result.resolution
             }
         }
     }
 
-    private fun tryFindNodeModule(): MutableSet<File> {
+    private fun tryFindNodeModule(): File? {
-        // todo: transitive dependencies
         var p = project
         do {
-            val result = p.rootDir.resolve("node_modules/$key")
+            val result = NpmProjectLayout[p].nodeModulesDir.resolve(key)
-            if (result.exists()) return mutableSetOf(result)
+            if (result.exists()) return result
-            p = project.parent ?: return mutableSetOf()
+            p = project.parent ?: return null
         } while (true)
     }
 
@@ -62,6 +85,13 @@ data class NpmDependency(
 
     override fun toString() = "$key: $version"
 
+    override fun resolve(context: DependencyResolveContext) {
+        resolve()
+        dependencies.forEach {
+            context.add(it)
+        }
+    }
+
     override fun resolve(transitive: Boolean): MutableSet<File> = resolve()
 
     override fun getFiles(): FileCollection = project.files(resolve())

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/NpmResolver.kt

@@ -54,10 +54,9 @@ internal class NpmResolver private constructor(val rootProject: Project) {
         class ResolvedNow(val resolution: ResolvedProject) : ResolutionCallResult()
     }
 
-    private val nodeJs = NodeJsPlugin.apply(rootProject).root
+    private val nodeJs = NodeJsPlugin.apply(rootProject)
-    private val npmProject = rootProject.npmProject
     private val packageManager = nodeJs.packageManager
-    private val hoistGradleNodeModules = npmProject.hoistGradleNodeModules
+    private val hoistGradleNodeModules = packageManager.getHoistGradleNodeModules(rootProject)
     private val npmPackages = mutableListOf<NpmPackage>()
     private val gson = GsonBuilder()
         .setPrettyPrinting()

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/npm/PackageJson.kt

@@ -10,11 +10,15 @@ class PackageJson(
     var version: String
 ) {
     val empty: Boolean
-        get() = private == null && workspaces == null && dependencies.isEmpty()
+        get() = private == null &&
+                workspaces == null &&
+                dependencies.isEmpty() &&
+                devDependencies.isEmpty()
 
     var private: Boolean? = null
 
     var workspaces: Collection<String>? = null
 
+    val devDependencies = mutableMapOf<String, String>()
     val dependencies = mutableMapOf<String, String>()
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/Yarn.kt

@@ -6,55 +6,119 @@
 package org.jetbrains.kotlin.gradle.targets.js.yarn
 
 import org.gradle.api.Project
+import org.gradle.api.internal.project.ProjectInternal
+import org.gradle.internal.hash.FileHasher
+import org.jetbrains.kotlin.daemon.common.toHexString
 import org.jetbrains.kotlin.gradle.internal.execWithProgress
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin
-import org.jetbrains.kotlin.gradle.targets.js.npm.NpmApi
+import org.jetbrains.kotlin.gradle.targets.js.npm.*
-import org.jetbrains.kotlin.gradle.targets.js.npm.NpmProjectLayout
+import java.io.File
-import org.jetbrains.kotlin.gradle.targets.js.npm.NpmResolver
-import org.jetbrains.kotlin.gradle.targets.js.npm.PackageJson
-import org.slf4j.LoggerFactory
 
 object Yarn : NpmApi {
-    private val log = LoggerFactory.getLogger("org.jetbrains.kotlin.gradle.targets.js.yarn.Yarn")
-
     override fun setup(project: Project) {
-        YarnPlugin[project].executeSetup()
+        YarnPlugin.apply(project).executeSetup()
     }
 
+    val Project.packageJsonHashFile: File
+        get() = buildDir.resolve("package.json.hash")
+
     fun yarnExec(
         project: Project,
         description: String,
         vararg args: String,
         npmProjectLayout: NpmProjectLayout = NpmProjectLayout[project]
     ) {
-        val nodeJsEnv = NodeJsPlugin[project].buildEnv()
+        val nodeJsEnv = NodeJsPlugin.apply(project).environment
-        val yarnEnv = YarnPlugin[project].buildEnv()
+        val yarnEnv = YarnPlugin.apply(project).environment
+
+        val packageJsonHashFile = project.packageJsonHashFile
+        val packageJsonHash = if (packageJsonHashFile.exists()) packageJsonHashFile.readText() else null
+
+        val hasher = (project as ProjectInternal).services.get(FileHasher::class.java)
+        val hash = hasher.hash(npmProjectLayout.packageJsonFile).toByteArray().toHexString()
+
+        if (packageJsonHash == hash) return
+
+        packageJsonHashFile.writeText(hash)
+
+        val nodeWorkDir = npmProjectLayout.nodeWorkDir
 
         project.execWithProgress(description) { exec ->
             exec.executable = nodeJsEnv.nodeExecutable
             exec.args = listOf(yarnEnv.home.resolve("bin/yarn.js").absolutePath) + args
-            exec.workingDir = npmProjectLayout.nodeWorkDir
+            exec.workingDir = nodeWorkDir
+        }
+    }
+
+    private fun yarnLockReadTransitiveDependencies(
+        nodeWorkDir: File,
+        srcDependenciesList: Collection<NpmDependency>
+    ) {
+        val yarnLock = nodeWorkDir.resolve("yarn.lock")
+        if (yarnLock.isFile) {
+            val byKey = YarnLock.parse(yarnLock).entries.associateBy { it.key }
+            val visited = mutableSetOf<NpmDependency>()
+
+            fun resolveRecursively(src: NpmDependency): NpmDependency {
+                if (src in visited) return src
+                visited.add(src)
+
+                val key = YarnLock.key(src.key, src.version)
+                val deps = byKey[key]
+                if (deps != null) {
+                    src.dependencies.addAll(deps.dependencies.map { dep ->
+                        resolveRecursively(
+                            NpmDependency(
+                                src.project,
+                                dep.group,
+                                dep.packageName,
+                                dep.version ?: "*"
+                            )
+                        )
+                    })
+                } else {
+                    // todo: [WARN] cannot find $key in yarn.lock
+                }
+
+                return src
+            }
+
+            srcDependenciesList.forEach { src ->
+                resolveRecursively(src)
+            }
         }
     }
 
     @Suppress("EXPOSED_PARAMETER_TYPE")
     override fun resolveProject(npmPackage: NpmResolver.NpmPackage) {
         val project = npmPackage.project
-        if (!YarnPlugin[project].checkUseWorkspace()) {
+        if (!project.yarn.useWorkspaces) {
             yarnExec(project, NpmApi.resolveOperationDescription("yarn for ${project.path}"))
+            yarnLockReadTransitiveDependencies(NpmProjectLayout[project].nodeWorkDir, npmPackage.npmDependencies)
         }
     }
 
     @Suppress("EXPOSED_PARAMETER_TYPE")
-    override fun hookRootPackage(rootProject: Project, rootPackageJson: PackageJson, allWorkspaces: Collection<NpmResolver.NpmPackage>) {
+    override fun hookRootPackage(
-        if (YarnPlugin[rootProject].checkUseWorkspace()) {
+        rootProject: Project,
+        rootPackageJson: PackageJson,
+        allWorkspaces: Collection<NpmResolver.NpmPackage>
+    ): Boolean {
+        if (rootProject.yarn.useWorkspaces) {
             rootPackageJson.private = true
             rootPackageJson.workspaces = allWorkspaces
                 .filter { it.project != rootProject }
-                .map { it.project.rootDir.relativeTo(rootProject.rootDir).path }
+                .map { it.project.projectDir.relativeTo(rootProject.rootDir).path }
+
+            return true
         }
+
+        return false
     }
 
+    override fun getHoistGradleNodeModules(project: Project): Boolean =
+        project.rootProject.yarn.useWorkspaces
+
     @Suppress("EXPOSED_PARAMETER_TYPE")
     override fun resolveRootProject(
         rootProject: Project,
@@ -62,8 +126,9 @@ object Yarn : NpmApi {
     ) {
         check(rootProject == rootProject.rootProject)
 
-        if (YarnPlugin[rootProject].checkUseWorkspace()) {
+        if (rootProject.yarn.useWorkspaces) {
             yarnExec(rootProject, NpmApi.resolveOperationDescription("yarn"))
+            yarnLockReadTransitiveDependencies(NpmProjectLayout[rootProject].nodeWorkDir, subprojects.flatMap { it.npmDependencies })
         } else {
             if (subprojects.any { it.project != rootProject }) {
                 // todo: proofread message
@@ -72,11 +137,15 @@ object Yarn : NpmApi {
                             "It is recommended to enable yarn workspaces to store common NPM dependencies in root project. " +
                             "To enable it add this to your root project: \n" +
                             "nodeJs { manageNodeModules = true } \n" +
-                            "yarn { useWorkspaces = true } \n" +
                             "Note: with `manageNodeModules` enabled, your `node_modules` and `package.json` files will be managed by " +
                             "Gradle, will be overridden during build and should be ignored in VCS."
                 )
             }
         }
     }
+
+    override fun cleanProject(project: Project) {
+        super.cleanProject(project)
+        project.packageJsonHashFile.delete()
+    }
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/yarn/YarnLock.kt

@@ -6,7 +6,6 @@
 package org.jetbrains.kotlin.gradle.targets.js.yarn
 
 import com.google.gson.stream.JsonReader
-import org.jetbrains.kotlin.gradle.targets.js.npm.PackageJson
 import java.io.File
 import java.io.StringReader
 
@@ -17,11 +16,23 @@ data class YarnLock(val entries: List<Entry>) {
         val resolved: String?,
         val integrity: String?,
         val dependencies: List<Dependency>
-    )
+    ) {
+        val packageKey = key.substringBeforeLast("@")
+    }
 
     data class Dependency(val key: String, val version: String?) {
-        val scopedName: PackageJson.ScopedName
+        val group: String?
-            get() = PackageJson.scopedName(key)
+        val packageName: String
+
+        init {
+            if (key.contains("/")) {
+                group = key.substringBeforeLast("/").removePrefix("@")
+                packageName = key.substringAfterLast("/")
+            } else {
+                group = null
+                packageName = key
+            }
+        }
     }
 
     companion object {

libraries/tools/kotlin-gradle-plugin/src/main/resources/META-INF/gradle-plugins/org.jetbrains.kotlin.nodejs.properties

@@ -0,0 +1 @@
+implementation-class=org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlugin