Gradle: introduce CleanabeStore

2019-12-20 15:39:37 +03:00
parent c0c929c449
commit 9c08ea3bf1
15 changed files with 342 additions and 13 deletions
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.tasks
+
+import org.jetbrains.kotlin.gradle.BaseGradleIT
+import org.jetbrains.kotlin.gradle.GradleVersionRequired
+import org.jetbrains.kotlin.gradle.transformProjectWithPluginsDsl
+import org.junit.Test
+
+
+class CleanDataTaskIT : BaseGradleIT() {
+
+    override val defaultGradleVersion: GradleVersionRequired
+        get() = GradleVersionRequired.AtLeast("5.5.1")
+
+    @Test
+    fun testDownloadedFolderDeletion() {
+        val project = transformProjectWithPluginsDsl("cleanTask")
+
+        project.build("testCleanTask") {
+            assertSuccessful()
+        }
+
+    }
+
+}
@@ -0,0 +1,68 @@
+import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlatform
+
+import java.io.File
+import java.time.*
+
+plugins {
+    id "org.jetbrains.kotlin.js" version "<pluginMarkerVersion>"
+}
+
+String nodeJsVersion
+String nodeJsLocation
+
+tasks.create("checkDownloadedFolder") {
+    description = "updated last modified date for downloaded folder"
+    dependsOn "build"
+    doLast {
+        println("check downloaded folder existance")
+        def downloadedFolder = new File(project.gradle.gradleUserHomeDir, nodeJsLocation)
+        if (!downloadedFolder.exists() || !downloadedFolder.isDirectory()) {
+            throw new InvalidUserDataException("Downloaded folder was not found")
+        }
+        println("try to update lastModifiedDate")
+        println(downloadedFolder.lastModified())
+        def successUpdated = downloadedFolder.setLastModified(LocalDateTime.now().minusDays(35).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli())
+        println(downloadedFolder.lastModified())
+        if (!successUpdated) {
+            throw new InvalidUserDataException("Unable to update last modified date")
+        }
+    }
+}
+
+tasks.create("testCleanTask") {
+    description = "check clean task behaviour"
+    doLast {
+        println("check deletion of downloaded folder")
+        def downloadedFolder = new File(project.gradle.gradleUserHomeDir, nodeJsLocation)
+        if (downloadedFolder.exists()) {
+            throw new InvalidUserDataException("Folder was not deleted")
+        }
+    }
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+}
+
+afterEvaluate {
+    project.kotlinNodeJs.nodeVersion = "10.15.3"
+
+    project.tasks.nodeKotlinClean.dependsOn(checkDownloadedFolder)
+    project.tasks.testCleanTask.dependsOn(nodeKotlinClean)
+
+    nodeJsVersion = project.extensions.getByName("kotlinNodeJs").nodeVersion
+    println("Node js version: $nodeJsVersion")
+
+    String platform = NodeJsPlatform.name
+    String architecture = NodeJsPlatform.architecture
+
+    nodeJsLocation = "/nodejs/node-v$nodeJsVersion-$platform-$architecture"
+    println("Use $nodeJsLocation location")
+}
+
+dependencies {
+    implementation "org.jetbrains.kotlin:kotlin-stdlib-js"
+}
+
+kotlin.target.browser {}
@@ -0,0 +1,8 @@
+pluginManagement {
+    repositories {
+        maven { url '<mavenLocalUrl>' }
+        gradlePluginPortal()
+    }
+}
+
+rootProject.name = 'cleanTask'
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>CleanTask</title>
+  <script src="CleanTask.js"></script>
+</head>
+<body>
+
+</body>
+</html>
@@ -0,0 +1,10 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+import kotlin.browser.document
+
+fun main() {
+    document.write("Hello, world!")
+}
@@ -8,6 +8,7 @@ import org.jetbrains.kotlin.gradle.targets.js.npm.KotlinNpmResolutionManager
 import org.jetbrains.kotlin.gradle.targets.js.npm.NpmApi
 import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinNpmInstallTask
 import org.jetbrains.kotlin.gradle.targets.js.yarn.Yarn
+import org.jetbrains.kotlin.gradle.tasks.internal.CleanableStore
 import java.io.File

 open class NodeJsRootExtension(val rootProject: Project) {
@@ -20,6 +21,7 @@ open class NodeJsRootExtension(val rootProject: Project) {
    }

    var installationDir = gradleHome.resolve("nodejs")
+    var cleanableStore = CleanableStore[installationDir.absolutePath]

    var download = true
    var nodeDownloadBaseUrl = "https://nodejs.org/dist"
@@ -64,7 +66,8 @@ open class NodeJsRootExtension(val rootProject: Project) {
            val platform = NodeJsPlatform.name
            val architecture = NodeJsPlatform.architecture

-            val nodeDir = installationDir.resolve("node-v$nodeVersion-$platform-$architecture")
+            val nodeDirName = "node-v$nodeVersion-$platform-$architecture"
+            val nodeDir = cleanableStore[nodeDirName].use()
            val isWindows = NodeJsPlatform.name == NodeJsPlatform.WIN
            val nodeBinDir = if (isWindows) nodeDir else nodeDir.resolve("bin")

@@ -3,9 +3,9 @@ package org.jetbrains.kotlin.gradle.targets.js.nodejs
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.plugins.BasePlugin
-import org.gradle.api.tasks.Delete
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension.Companion.EXTENSION_NAME
 import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinNpmInstallTask
+import org.jetbrains.kotlin.gradle.tasks.CleanDataTask

 open class NodeJsRootPlugin : Plugin<Project> {
    override fun apply(project: Project): Unit = project.run {
@@ -15,7 +15,7 @@ open class NodeJsRootPlugin : Plugin<Project> {
            "NodeJsRootPlugin can be applied only to root project"
        }

-        this.extensions.create(EXTENSION_NAME, NodeJsRootExtension::class.java, this)
+        val settings = this.extensions.create(EXTENSION_NAME, NodeJsRootExtension::class.java, this)

        val setupTask = tasks.create(NodeJsSetupTask.NAME, NodeJsSetupTask::class.java) {
            it.group = TASKS_GROUP_NAME
@@ -27,6 +27,12 @@ open class NodeJsRootPlugin : Plugin<Project> {
            it.group = TASKS_GROUP_NAME
            it.description = "Find, download and link NPM dependencies and projects"
        }
+
+        tasks.create("node" + CleanDataTask.NAME, CleanDataTask::class.java) {
+            it.cleanableStore = settings.cleanableStore
+            it.group = TASKS_GROUP_NAME
+            it.description = "Clean unused local node version"
+        }
    }

    companion object {
@@ -25,7 +25,8 @@ abstract class YarnBasics : NpmApi {
    ) {
        val nodeJs = NodeJsRootPlugin.apply(project)
        val nodeJsEnv = nodeJs.environment
-        val yarnEnv = YarnPlugin.apply(project).environment
+        val yarnPlugin = YarnPlugin.apply(project)
+        val yarnEnv = yarnPlugin.environment

        project.execWithProgress(description) { exec ->
            exec.executable = nodeJsEnv.nodeExecutable
@@ -34,6 +35,7 @@ abstract class YarnBasics : NpmApi {
                    if (project.logger.isDebugEnabled) "--verbose" else ""
            exec.workingDir = dir
        }
+
    }

    protected fun yarnLockReadTransitiveDependencies(
@@ -8,6 +8,8 @@ package org.jetbrains.kotlin.gradle.targets.js.yarn
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootPlugin
+import org.jetbrains.kotlin.gradle.tasks.CleanDataTask
+import org.jetbrains.kotlin.gradle.tasks.internal.CleanableStore

 open class YarnPlugin : Plugin<Project> {
    override fun apply(project: Project): Unit = project.run {
@@ -17,10 +19,16 @@ open class YarnPlugin : Plugin<Project> {

        val nodeJs = NodeJsRootPlugin.apply(this)

-        this.extensions.create(YarnRootExtension.YARN, YarnRootExtension::class.java, this)
+        val yarnRootExtension = this.extensions.create(YarnRootExtension.YARN, YarnRootExtension::class.java, this)
+
        tasks.create(YarnSetupTask.NAME, YarnSetupTask::class.java) {
            it.dependsOn(nodeJs.nodeJsSetupTask)
        }
+
+        tasks.create("yarn" + CleanDataTask.NAME, CleanDataTask::class.java) {
+            it.cleanableStore = yarnRootExtension.cleanableStore
+            it.description = "Clean unused local yarn version"
+        }
    }

    companion object {
@@ -8,6 +8,7 @@ package org.jetbrains.kotlin.gradle.targets.js.yarn
 import org.gradle.api.Project
 import org.jetbrains.kotlin.gradle.logging.kotlinInfo
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootPlugin
+import org.jetbrains.kotlin.gradle.tasks.internal.CleanableStore

 open class YarnRootExtension(val project: Project) {
    init {
@@ -20,6 +21,9 @@ open class YarnRootExtension(val project: Project) {

    var installationDir = gradleHome.resolve("yarn")

+    // TODO: Split configuration and execution phases to support changed installationDir
+    val cleanableStore = CleanableStore[installationDir.path]
+
    var downloadBaseUrl = "https://github.com/yarnpkg/yarn/releases/download"
    var version = "1.21.1"

@@ -31,21 +35,20 @@ open class YarnRootExtension(val project: Project) {
    val useWorkspaces: Boolean
        get() = !disableWorkspaces

+    // TODO: Split configuration and execution phases to support changed settings
+    internal val environment = YarnEnv(
+        downloadUrl = "$downloadBaseUrl/v$version/yarn-v$version.tar.gz",
+        home = cleanableStore["yarn-v$version"].use()
+    )
+
    internal fun executeSetup() {
        NodeJsRootPlugin.apply(project).executeSetup()

-        val env = environment
-        if (!env.home.isDirectory) {
+        if (!environment.home.isDirectory) {
            yarnSetupTask.setup()
        }
    }

-    internal val environment
-        get() = YarnEnv(
-            downloadUrl = "$downloadBaseUrl/v$version/yarn-v$version.tar.gz",
-            home = installationDir.resolve("yarn-v$version")
-        )
-
    companion object {
        const val YARN: String = "kotlinYarn"

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.tasks
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.TaskAction
+import org.jetbrains.kotlin.gradle.tasks.internal.CleanableStore
+import java.time.Duration
+import java.time.Instant
+
+/**
+ * Task to clean all old loaded files based on a date of special access file.
+ * The access file should be updated every time when loaded files are used
+ */
+open class CleanDataTask : DefaultTask() {
+
+    /**
+     * Path to folder.
+     * Use path instead of file to avoid file scanning for change check
+     */
+    @Input
+    lateinit var cleanableStore: CleanableStore
+
+    /**
+     * Time to live in days
+     */
+    @Input
+    var timeToLiveInDays: Long = 30
+
+    @Suppress("unused")
+    @TaskAction
+    fun exec() {
+        val expirationDate = Instant.now().minus(Duration.ofDays(timeToLiveInDays))
+
+        cleanableStore.cleanDir(expirationDate)
+
+    }
+
+    companion object {
+        const val NAME: String = "KotlinClean"
+    }
+
+}
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.tasks
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.tasks.Input
+import org.gradle.api.tasks.TaskAction
+import org.jetbrains.kotlin.gradle.tasks.internal.CleanableStore
+import java.time.*
+
+/**
+ * Task to clean all old loaded files based on a last modification date.
+ * All registered store in {@link CleanableStore} would be cleaned
+ */
+class CleanOldStoredDataTask : DefaultTask() {
+
+    /**
+     * Time to live in days
+     */
+    @Input
+    val timeToLiveInDays: Long = 30
+
+    @Suppress("unused")
+    @TaskAction
+    fun exec() {
+        val expirationDate = Instant.now().minus(Duration.ofDays(timeToLiveInDays))
+
+        CleanableStore.stores.forEach { (_, store) -> store.cleanDir(expirationDate) }
+
+    }
+
+    companion object {
+        const val NAME: String = "clean store"
+    }
+
+}
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.tasks.internal
+
+import org.jetbrains.kotlin.gradle.tasks.CleanDataTask
+import java.time.Instant
+
+/**
+ * Store of tracked directories that can be cleaned with [CleanDataTask].
+ * All directories that was not marked as used at least [CleanDataTask.timeToLiveInDays] days will be removed.
+ *
+ * To register store call `CleanableStore["/path/to/dir"]`.
+ * Now you will be able to access files via `CleanableStore["/path/to/dir"]["file/name"].use()`
+ * and it would update usage of th store.
+ */
+interface CleanableStore {
+    fun cleanDir(expirationDate: Instant)
+
+    operator fun get(fileName: String): DownloadedFile
+
+    fun markUsed()
+
+    companion object {
+        private val mutableStores = mutableMapOf<String, CleanableStore>()
+
+        val stores: Map<String, CleanableStore>
+            get() = mutableStores.toMap()
+
+        operator fun get(path: String): CleanableStore =
+            mutableStores.getOrPut(path) { CleanableStoreImpl(path) }
+    }
+}
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.tasks.internal
+
+import org.gradle.util.GFileUtils
+import java.io.File
+import java.nio.file.Files
+import java.time.Instant
+
+internal class CleanableStoreImpl(dirPath: String) : CleanableStore {
+    private val dir = File(dirPath)
+
+    override fun get(fileName: String): DownloadedFile =
+        DownloadedFile(this, dir.resolve(fileName))
+
+    override fun markUsed() {
+        if (dir.exists()) {
+            GFileUtils.touchExisting(dir)
+        }
+    }
+
+    override fun cleanDir(expirationDate: Instant) {
+        fun modificationDate(file: File): Instant {
+            return Files.getLastModifiedTime(file.toPath()).toInstant()
+        }
+
+        dir.listFiles()
+            ?.filter { file ->
+                modificationDate(file).isBefore(expirationDate)
+            }
+            ?.forEach { file -> file.deleteRecursively() }
+    }
+}
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.tasks.internal
+
+import java.io.File
+
+/**
+ * Wrapper around [File] to be able to mark [store] storage as used by calling [use]`()`.
+ */
+class DownloadedFile internal constructor(
+    private val store: CleanableStore,
+    private val file: File
+) {
+    fun use(): File {
+        store.markUsed()
+        return file
+    }
+
+    fun resolve(fileName: String): DownloadedFile =
+        DownloadedFile(store, file.resolve(fileName))
+}