[Gradle, JS] Update detecting of platform and architecture

[Gradle, JS] Make hash codes compatible with previous kotlin ^KT-38015 fixed ^KT-48631 fixed ^KT-43305 fixed
2023-01-19 10:23:09 +00:00
parent b1bb7dd25f
commit 843805d771
8 changed files with 289 additions and 151 deletions
@@ -1,5 +1,3 @@
-import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlatform
-
 import java.io.File
 import java.time.*
 import java.util.concurrent.*
@@ -68,6 +66,71 @@ repositories {
    mavenCentral()
 }

+class NodeJsPlatform {
+    def props = System.getProperties()
+    String property(String name) {
+        return props.getProperty(name) ?: System.getProperty(name)
+    }
+
+    def WIN = "win"
+    def DARWIN = "darwin"
+    def LINUX = "linux"
+    def SUNOS = "sunos"
+
+    String name = {
+        def name = property("os.name").toLowerCase()
+        if (name.contains("windows")) {
+            return WIN
+        }
+
+        if (name.contains("mac")) {
+            return DARWIN
+        }
+
+        if (name.contains("linux")) {
+            return LINUX
+        }
+
+        if (name.contains("sunos")) {
+            return SUNOS
+        }
+
+        throw new IllegalArgumentException("Unsupported OS: $name")
+    }()
+
+    def X64 = "x64"
+    def X86 = "x86"
+    def ARM64 = "arm64"
+
+    String architecture = {
+        def arch = property("os.arch").toLowerCase()
+        if (arch == "aarch64") {
+            return ARM64
+        }
+        if (arch.contains("64")) {
+            return X64
+        }
+        if (arch == "arm") {
+            // as Java just returns "arm" on all ARM variants, we need a system call to determine the exact arch
+            // the node binaries for 'armv8l' are called 'arm64', so we need to distinguish here
+            def process = Runtime.getRuntime().exec("uname -m")
+            def is = process.getInputStream()
+            String systemArch = null
+            is.eachLine {
+                systemArch = it
+            }
+
+            if (systemArch.trim() == "armv8l") {
+                return ARM64
+            } else {
+                return systemArch
+            }
+        }
+
+        return X86
+    }()
+}
+
 afterEvaluate {
    project.kotlinNodeJs.nodeVersion = "14.16.0"

@@ -77,8 +140,10 @@ afterEvaluate {
    nodeJsVersion = project.extensions.getByName("kotlinNodeJs").nodeVersion
    println("Node js version: $nodeJsVersion")

-    String platform = NodeJsPlatform.name
-    String architecture = NodeJsPlatform.architecture
+    def nodeJsPlatform = new NodeJsPlatform()
+
+    String platform = nodeJsPlatform.name
+    String architecture = nodeJsPlatform.architecture

    nodeJsLocation = "/nodejs/node-v$nodeJsVersion-$platform-$architecture"
    println("Use $nodeJsLocation location")
@@ -1,52 +0,0 @@
-package org.jetbrains.kotlin.gradle.targets.js.nodejs
-
-import org.jetbrains.kotlin.util.capitalizeDecapitalize.toLowerCaseAsciiOnly
-
-/**
- * Provides platform and architecture names that is used to download NodeJs.
- * See [NodeJsEnv.ivyDependency] that is filled in [NodeJsRootExtension.environment].
- */
-internal object NodeJsPlatform {
-    private val props = System.getProperties()
-    private fun property(name: String) = props.getProperty(name) ?: System.getProperty(name)
-
-    const val WIN = "win"
-    const val DARWIN = "darwin"
-    const val LINUX = "linux"
-    const val SUNOS = "sunos"
-
-    val name: String = run {
-        val name = property("os.name").toLowerCaseAsciiOnly()
-        when {
-            name.contains("windows") -> WIN
-            name.contains("mac") -> DARWIN
-            name.contains("linux") -> LINUX
-            name.contains("freebsd") -> LINUX
-            name.contains("sunos") -> SUNOS
-            else -> throw IllegalArgumentException("Unsupported OS: $name")
-        }
-    }
-
-    const val X64 = "x64"
-    const val X86 = "x86"
-    const val ARM64 = "arm64"
-
-    val architecture: String = run {
-        val arch = property("os.arch").toLowerCaseAsciiOnly()
-        when {
-            arch == "aarch64" -> ARM64
-            arch.contains("64") -> X64
-            arch == "arm" -> {
-                // as Java just returns "arm" on all ARM variants, we need a system call to determine the exact arch
-                // the node binaries for 'armv8l' are called 'arm64', so we need to distinguish here
-                val process = Runtime.getRuntime().exec("uname -m")
-                val systemArch = process.inputStream.bufferedReader().use { it.readText() }
-                when (systemArch.trim()) {
-                    "armv8l" -> ARM64
-                    else -> systemArch
-                }
-            }
-            else -> X86
-        }
-    }
-}
@@ -75,26 +75,41 @@ open class NodeJsRootExtension(@Transient val rootProject: Project) : Configurat
    val nodeJsSetupTaskProvider: TaskProvider<out NodeJsSetupTask>
        get() = rootProject.tasks.withType(NodeJsSetupTask::class.java).named(NodeJsSetupTask.NAME)

-    @Suppress("UNNECESSARY_SAFE_CALL", "SAFE_CALL_WILL_CHANGE_NULLABILITY") // TODO: investigate this warning; fixing it breaks integration tests.
+    @Suppress(
+        "UNNECESSARY_SAFE_CALL",
+        "SAFE_CALL_WILL_CHANGE_NULLABILITY"
+    ) // TODO: investigate this warning; fixing it breaks integration tests.
    val npmInstallTaskProvider: TaskProvider<out KotlinNpmInstallTask>?
        get() = rootProject?.tasks?.withType(KotlinNpmInstallTask::class.java)?.named(KotlinNpmInstallTask.NAME)

    val packageJsonUmbrellaTaskProvider: TaskProvider<Task>
        get() = rootProject.tasks.named(PACKAGE_JSON_UMBRELLA_TASK_NAME)

-    @Suppress("UNNECESSARY_SAFE_CALL", "SAFE_CALL_WILL_CHANGE_NULLABILITY") // TODO: investigate this warning; fixing it breaks integration tests.
+    @Suppress(
+        "UNNECESSARY_SAFE_CALL",
+        "SAFE_CALL_WILL_CHANGE_NULLABILITY"
+    ) // TODO: investigate this warning; fixing it breaks integration tests.
    val rootPackageJsonTaskProvider: TaskProvider<RootPackageJsonTask>?
        get() = rootProject?.tasks?.withType(RootPackageJsonTask::class.java)?.named(RootPackageJsonTask.NAME)

-    @Suppress("UNNECESSARY_SAFE_CALL", "SAFE_CALL_WILL_CHANGE_NULLABILITY") // TODO: investigate this warning; fixing it breaks integration tests.
+    @Suppress(
+        "UNNECESSARY_SAFE_CALL",
+        "SAFE_CALL_WILL_CHANGE_NULLABILITY"
+    ) // TODO: investigate this warning; fixing it breaks integration tests.
    val npmCachesSetupTaskProvider: TaskProvider<out KotlinNpmCachesSetup>?
        get() = rootProject?.tasks?.withType(KotlinNpmCachesSetup::class.java)?.named(KotlinNpmCachesSetup.NAME)

-    @Suppress("UNNECESSARY_SAFE_CALL", "SAFE_CALL_WILL_CHANGE_NULLABILITY") // TODO: investigate this warning; fixing it breaks integration tests.
+    @Suppress(
+        "UNNECESSARY_SAFE_CALL",
+        "SAFE_CALL_WILL_CHANGE_NULLABILITY"
+    ) // TODO: investigate this warning; fixing it breaks integration tests.
    val storeYarnLockTaskProvider: TaskProvider<out YarnLockCopyTask>?
        get() = rootProject?.tasks?.withType(YarnLockCopyTask::class.java)?.named(STORE_YARN_LOCK_NAME)

-    @Suppress("UNNECESSARY_SAFE_CALL", "SAFE_CALL_WILL_CHANGE_NULLABILITY") // TODO: investigate this warning; fixing it breaks integration tests.
+    @Suppress(
+        "UNNECESSARY_SAFE_CALL",
+        "SAFE_CALL_WILL_CHANGE_NULLABILITY"
+    ) // TODO: investigate this warning; fixing it breaks integration tests.
    val restoreYarnLockTaskProvider: TaskProvider<out YarnLockCopyTask>?
        get() = rootProject?.tasks?.withType(YarnLockCopyTask::class.java)?.named(RESTORE_YARN_LOCK_NAME)

@@ -109,13 +124,14 @@ open class NodeJsRootExtension(@Transient val rootProject: Project) : Configurat
        get() = rootPackageDir.resolve("packages_imported")

    override fun finalizeConfiguration(): NodeJsEnv {
-        val platform = NodeJsPlatform.name
-        val architecture = NodeJsPlatform.architecture
+        val platformHelper = PlatformHelper
+        val platform = platformHelper.osName
+        val architecture = platformHelper.osArch

        val nodeDirName = "node-v$nodeVersion-$platform-$architecture"
        val cleanableStore = CleanableStore[installationDir.absolutePath]
        val nodeDir = cleanableStore[nodeDirName].use()
-        val isWindows = NodeJsPlatform.name == NodeJsPlatform.WIN
+        val isWindows = platformHelper.isWindows
        val nodeBinDir = if (isWindows) nodeDir else nodeDir.resolve("bin")

        fun getExecutable(command: String, customCommand: String, windowsExtension: String): String {
@@ -3,17 +3,20 @@ package org.jetbrains.kotlin.gradle.targets.js.nodejs
 import org.gradle.api.DefaultTask
 import org.gradle.api.artifacts.Configuration
 import org.gradle.api.file.FileSystemOperations
-import org.gradle.api.file.FileTree
+import org.gradle.api.model.ObjectFactory
 import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.*
 import org.gradle.internal.hash.FileHasher
 import org.jetbrains.kotlin.gradle.logging.kotlinInfo
 import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService
-import org.jetbrains.kotlin.gradle.targets.js.calculateDirHash
+import org.jetbrains.kotlin.gradle.targets.js.extractWithUpToDate
 import org.jetbrains.kotlin.gradle.utils.ArchiveOperationsCompat
 import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
 import java.io.File
 import java.net.URI
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
 import javax.inject.Inject

 abstract class NodeJsSetupTask : DefaultTask() {
@@ -29,6 +32,9 @@ abstract class NodeJsSetupTask : DefaultTask() {
    internal open val fileHasher: FileHasher
        get() = error("Should be injected")

+    @get:Inject
+    internal abstract val objects: ObjectFactory
+
    @get:Inject
    internal open val fs: FileSystemOperations
        get() = error("Should be injected")
@@ -87,57 +93,55 @@ abstract class NodeJsSetupTask : DefaultTask() {
        if (!shouldDownload) return
        logger.kotlinInfo("Using node distribution from '$nodeJsDist'")

-        var dirHash: String? = null
-        val upToDate = destinationHashFile.let { file ->
-            if (file.exists()) {
-                file.useLines {
-                    it.single() == (fileHasher.calculateDirHash(destination).also { dirHash = it })
-                }
-            } else false
-        }
+        extractWithUpToDate(
+            destination,
+            destinationHashFile,
+            nodeJsDist!!,
+            fileHasher
+        ) { dist, destination ->
+            var fixBrokenSymLinks = false

-        val tmpDir = temporaryDir
-        unpackNodeArchive(nodeJsDist!!, tmpDir)
+            fs.copy {
+                it.from(
+                    when {
+                        dist.name.endsWith("zip") -> archiveOperations.zipTree(dist)
+                        else -> {
+                            fixBrokenSymLinks = true
+                            archiveOperations.tarTree(dist)
+                        }
+                    }
+                )
+                it.into(destination)
+            }

-        if (upToDate && fileHasher.calculateDirHash(tmpDir.resolve(destination.name))!! == dirHash) {
-            tmpDir.deleteRecursively()
-            return
-        }
+            fixBrokenSymlinks(destination, env.isWindows, fixBrokenSymLinks)

-        if (destination.isDirectory) {
-            destination.deleteRecursively()
-        }
-
-        fs.copy {
-            it.from(tmpDir)
-            it.into(destination.parentFile)
-        }
-
-        tmpDir.deleteRecursively()
-
-        if (!env.isWindows) {
-            File(env.nodeExecutable).setExecutable(true)
-        }
-
-        destinationHashFile.writeText(
-            fileHasher.calculateDirHash(destination)!!
-        )
-    }
-
-    private fun unpackNodeArchive(archive: File, destination: File) {
-        logger.kotlinInfo("Unpacking $archive to $destination")
-
-        fs.copy {
-            it.from(fileTree(archive))
-            it.into(destination)
+            if (!env.isWindows) {
+                File(env.nodeExecutable).setExecutable(true)
+            }
        }
    }

-    private fun fileTree(archive: File): FileTree =
-        when {
-            archive.name.endsWith("zip") -> archiveOperations.zipTree(archive)
-            else -> archiveOperations.tarTree(archive)
+    private fun fixBrokenSymlinks(destinationDir: File, isWindows: Boolean, necessaryToFix: Boolean) {
+        if (necessaryToFix) {
+            val nodeBinDir = computeNodeBinDir(destinationDir, isWindows).toPath()
+            fixBrokenSymlink("npm", nodeBinDir, destinationDir, isWindows)
+            fixBrokenSymlink("npx", nodeBinDir, destinationDir, isWindows)
        }
+    }
+
+    private fun fixBrokenSymlink(
+        name: String,
+        nodeBinDirPath: Path,
+        nodeDirProvider: File,
+        isWindows: Boolean
+    ) {
+        val script = nodeBinDirPath.resolve(name)
+        val scriptFile = computeNpmScriptFile(nodeDirProvider, name, isWindows)
+        if (Files.deleteIfExists(script)) {
+            Files.createSymbolicLink(script, nodeBinDirPath.relativize(Paths.get(scriptFile)))
+        }
+    }

    companion object {
        const val NAME: String = "kotlinNodeJsSetup"
@@ -0,0 +1,91 @@
+package org.jetbrains.kotlin.gradle.targets.js.nodejs
+
+import java.io.File
+import java.util.concurrent.TimeUnit
+
+object PlatformHelper {
+    val osName: String by lazy {
+        val name = property("os.name").toLowerCase()
+        when {
+            name.contains("windows") -> "win"
+            name.contains("mac") -> "darwin"
+            name.contains("linux") -> "linux"
+            name.contains("freebsd") -> "linux"
+            name.contains("sunos") -> "sunos"
+            else -> error("Unsupported OS: $name")
+        }
+    }
+
+
+    val osArch: String by lazy {
+        val arch = property("os.arch").toLowerCase()
+        when {
+            /*
+             * As Java just returns "arm" on all ARM variants, we need a system call to determine the exact arch. Unfortunately some JVMs say aarch32/64, so we need an additional
+             * conditional. Additionally, the node binaries for 'armv8l' are called 'arm64', so we need to distinguish here.
+             */
+            arch == "arm" || arch.startsWith("aarch") -> property("uname")
+                .let {
+                    if (it == "armv8l" || it == "aarch64") {
+                        "arm64"
+                    } else it
+                }
+                .let {
+                    if (it == "x86_64") {
+                        "x64"
+                    } else it
+                }
+            arch == "ppc64le" -> "ppc64le"
+            arch == "s390x" -> "s390x"
+            arch.contains("64") -> "x64"
+            else -> "x86"
+        }
+    }
+
+
+    val isWindows: Boolean by lazy { osName == "win" }
+
+
+    private fun property(name: String): String {
+        return getSystemProperty(name) ?:
+        // Added so that we can test osArch on Windows and on non-arm systems
+        if (name == "uname") execute("uname", "-m")
+        else error("Unable to find a value for property [$name].")
+    }
+
+
+    private fun getSystemProperty(name: String): String? {
+        return System.getProperty(name);
+    }
+}
+
+fun computeNpmScriptFile(
+    nodeDirProvider: File,
+    command: String,
+    isWindows: Boolean
+): String {
+    return nodeDirProvider.let { nodeDir ->
+        if (isWindows) nodeDir.resolve("node_modules/npm/bin/$command-cli.js").path
+        else nodeDir.resolve("lib/node_modules/npm/bin/$command-cli.js").path
+    }
+}
+
+fun computeNodeBinDir(
+    nodeDirProvider: File,
+    isWindows: Boolean
+) =
+    if (isWindows) nodeDirProvider else nodeDirProvider.resolve("bin")
+
+/**
+ * Executes the given command and returns its output.
+ *
+ * This is based on an aggregate of the answers provided here: [https://stackoverflow.com/questions/35421699/how-to-invoke-external-command-from-within-kotlin-code]
+ */
+private fun execute(vararg args: String, timeout: Long = 60): String {
+    return ProcessBuilder(args.toList())
+        .redirectOutput(ProcessBuilder.Redirect.PIPE)
+        .redirectError(ProcessBuilder.Redirect.PIPE)
+        .start()
+        .apply { waitFor(timeout, TimeUnit.SECONDS) }
+        .inputStream.bufferedReader().readText().trim()
+}
@@ -7,11 +7,9 @@ package org.jetbrains.kotlin.gradle.targets.js

 import org.gradle.internal.hash.FileHasher
 import org.gradle.internal.hash.Hashing.defaultFunction
-import org.jetbrains.kotlin.gradle.internal.testing.TCServiceMessagesTestExecutor
 import org.jetbrains.kotlin.gradle.utils.appendLine
-import org.jetbrains.kotlin.gradle.utils.isConfigurationCacheAvailable
 import java.io.File
-import org.gradle.api.Project
+import java.nio.file.Files

 fun Appendable.appendConfigsFromDir(confDir: File) {
    val files = confDir.listFiles() ?: return
@@ -39,6 +37,42 @@ fun ByteArray.toHex(): String {
    return String(result)
 }

+fun extractWithUpToDate(
+    destination: File,
+    destinationHashFile: File,
+    dist: File,
+    fileHasher: FileHasher,
+    extract: (File, File) -> Unit
+) {
+    var distHash: String? = null
+    val upToDate = destinationHashFile.let { file ->
+        if (file.exists()) {
+            file.useLines { seq ->
+                val list = seq.first().split(" ")
+                list.size == 2 &&
+                        list[0] == fileHasher.calculateDirHash(destination) &&
+                        list[1] == fileHasher.hash(dist).toByteArray().toHex().also { distHash = it }
+            }
+        } else false
+    }
+
+    if (upToDate) {
+        return
+    }
+
+    if (destination.isDirectory) {
+        destination.deleteRecursively()
+    }
+
+    extract(dist, destination.parentFile)
+
+    destinationHashFile.writeText(
+        fileHasher.calculateDirHash(destination)!! +
+                " " +
+                (distHash ?: fileHasher.hash(dist).toByteArray().toHex())
+    )
+}
+
 fun FileHasher.calculateDirHash(
    dir: File
 ): String? {
@@ -48,14 +82,18 @@ fun FileHasher.calculateDirHash(
    dir.walk()
        .forEach { file ->
            hasher.putString(file.toRelativeString(dir))
-            if (file.isFile) {
-                hasher.putHash(hash(file))
+            if (file.isFile && !Files.isSymbolicLink(file.toPath())) {
+                if (!Files.isSymbolicLink(file.toPath())) {
+                    hasher.putHash(hash(file))
+                } else {
+                    val canonicalFile = file.canonicalFile
+                    hasher.putHash(hash(canonicalFile))
+                    hasher.putString(canonicalFile.toRelativeString(dir))
+                }
            }
        }

-    val digest = hasher.hash().toByteArray()
-
-    return digest.toHex()
+    return hasher.hash().toByteArray().toHex()
 }

 const val JS = "js"
@@ -5,13 +5,13 @@

 package org.jetbrains.kotlin.gradle.targets.js.yarn

+import org.jetbrains.kotlin.gradle.targets.js.nodejs.PlatformHelper
 import org.gradle.api.Action
 import org.gradle.api.Incubating
 import org.gradle.api.Project
 import org.gradle.api.tasks.TaskProvider
 import org.jetbrains.kotlin.gradle.internal.ConfigurationPhaseAware
 import org.jetbrains.kotlin.gradle.logging.kotlinInfo
-import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlatform
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootPlugin
 import org.jetbrains.kotlin.gradle.targets.js.npm.RequiresNpmDependencies
 import org.jetbrains.kotlin.gradle.targets.js.npm.resolver.implementing
@@ -93,7 +93,7 @@ open class YarnRootExtension(
    override fun finalizeConfiguration(): YarnEnv {
        val cleanableStore = CleanableStore[installationDir.path]

-        val isWindows = NodeJsPlatform.name == NodeJsPlatform.WIN
+        val isWindows = PlatformHelper.isWindows

        val home = cleanableStore["yarn-v$version"].use()

@@ -13,7 +13,7 @@ import org.gradle.api.tasks.*
 import org.gradle.internal.hash.FileHasher
 import org.jetbrains.kotlin.gradle.logging.kotlinInfo
 import org.jetbrains.kotlin.gradle.plugin.statistics.KotlinBuildStatsService
-import org.jetbrains.kotlin.gradle.targets.js.calculateDirHash
+import org.jetbrains.kotlin.gradle.targets.js.extractWithUpToDate
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootPlugin
 import org.jetbrains.kotlin.gradle.utils.ArchiveOperationsCompat
 import org.jetbrains.kotlin.statistics.metrics.NumericalMetrics
@@ -97,36 +97,12 @@ open class YarnSetupTask : DefaultTask() {
        if (!shouldDownload) return
        logger.kotlinInfo("Using yarn distribution from '$yarnDist'")

-        var dirHash: String? = null
-        val upToDate = destinationHashFile.let { file ->
-            if (file.exists()) {
-                file.useLines {
-                    it.single() == (fileHasher.calculateDirHash(destination).also { dirHash = it })
-                }
-            } else false
-        }
-
-        val tmpDir = temporaryDir
-        extract(yarnDist!!, tmpDir) // parent because archive contains name already
-
-        if (upToDate && fileHasher.calculateDirHash(tmpDir.resolve(destination.name))!! == dirHash) {
-            tmpDir.deleteRecursively()
-            return
-        }
-
-        if (destination.isDirectory) {
-            destination.deleteRecursively()
-        }
-
-        fs.copy {
-            it.from(tmpDir)
-            it.into(destination.parentFile)
-        }
-
-        tmpDir.deleteRecursively()
-
-        destinationHashFile.writeText(
-            fileHasher.calculateDirHash(destination)!!
+        extractWithUpToDate(
+            destination,
+            destinationHashFile,
+            yarnDist!!,
+            fileHasher,
+            ::extract
        )
    }