Gradle: introduce CleanabeStore

2019-12-20 15:39:37 +03:00
parent c0c929c449
commit 9c08ea3bf1
15 changed files with 342 additions and 13 deletions
@@ -0,0 +1,29 @@
 /*
 * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
 * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
 */
 package org.jetbrains.kotlin.gradle.tasks
 import org.jetbrains.kotlin.gradle.BaseGradleIT
 import org.jetbrains.kotlin.gradle.GradleVersionRequired
 import org.jetbrains.kotlin.gradle.transformProjectWithPluginsDsl
 import org.junit.Test
 class CleanDataTaskIT : BaseGradleIT() {
    override val defaultGradleVersion: GradleVersionRequired
        get() = GradleVersionRequired.AtLeast("5.5.1")
    @Test
    fun testDownloadedFolderDeletion() {
        val project = transformProjectWithPluginsDsl("cleanTask")
        project.build("testCleanTask") {
            assertSuccessful()
        }
    }
 }
@@ -0,0 +1,68 @@
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsPlatform
 import java.io.File
 import java.time.*
 plugins {
    id "org.jetbrains.kotlin.js" version "<pluginMarkerVersion>"
 }
 String nodeJsVersion
 String nodeJsLocation
 tasks.create("checkDownloadedFolder") {
    description = "updated last modified date for downloaded folder"
    dependsOn "build"
    doLast {
        println("check downloaded folder existance")
        def downloadedFolder = new File(project.gradle.gradleUserHomeDir, nodeJsLocation)
        if (!downloadedFolder.exists() || !downloadedFolder.isDirectory()) {
            throw new InvalidUserDataException("Downloaded folder was not found")
        }
        println("try to update lastModifiedDate")
        println(downloadedFolder.lastModified())
        def successUpdated = downloadedFolder.setLastModified(LocalDateTime.now().minusDays(35).atZone(ZoneId.systemDefault()).toInstant().toEpochMilli())
        println(downloadedFolder.lastModified())
        if (!successUpdated) {
            throw new InvalidUserDataException("Unable to update last modified date")
        }
    }
 }
 tasks.create("testCleanTask") {
    description = "check clean task behaviour"
    doLast {
        println("check deletion of downloaded folder")
        def downloadedFolder = new File(project.gradle.gradleUserHomeDir, nodeJsLocation)
        if (downloadedFolder.exists()) {
            throw new InvalidUserDataException("Folder was not deleted")
        }
    }
 }
 repositories {
    mavenLocal()
    mavenCentral()
 }
 afterEvaluate {
    project.kotlinNodeJs.nodeVersion = "10.15.3"
    project.tasks.nodeKotlinClean.dependsOn(checkDownloadedFolder)
    project.tasks.testCleanTask.dependsOn(nodeKotlinClean)
    nodeJsVersion = project.extensions.getByName("kotlinNodeJs").nodeVersion
    println("Node js version: $nodeJsVersion")
    String platform = NodeJsPlatform.name
    String architecture = NodeJsPlatform.architecture
    nodeJsLocation = "/nodejs/node-v$nodeJsVersion-$platform-$architecture"
    println("Use $nodeJsLocation location")
 }
 dependencies {
    implementation "org.jetbrains.kotlin:kotlin-stdlib-js"
 }
 kotlin.target.browser {}
@@ -0,0 +1,8 @@
 pluginManagement {
    repositories {
        maven { url '<mavenLocalUrl>' }
        gradlePluginPortal()
    }
 }
 rootProject.name = 'cleanTask'
@@ -0,0 +1,11 @@
 <!DOCTYPE html>
 <html lang="en">
 <head>
  <meta charset="UTF-8">
  <title>CleanTask</title>
  <script src="CleanTask.js"></script>
 </head>
 <body>
 </body>
 </html>
@@ -0,0 +1,10 @@
 /*
 * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
 * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
 */
 import kotlin.browser.document
 fun main() {
    document.write("Hello, world!")
 }
@@ -8,6 +8,7 @@ import org.jetbrains.kotlin.gradle.targets.js.npm.KotlinNpmResolutionManager
 import org.jetbrains.kotlin.gradle.targets.js.npm.NpmApi
 import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinNpmInstallTask
 import org.jetbrains.kotlin.gradle.targets.js.yarn.Yarn
 import org.jetbrains.kotlin.gradle.tasks.internal.CleanableStore
 import java.io.File
 open class NodeJsRootExtension(val rootProject: Project) {
@@ -20,6 +21,7 @@ open class NodeJsRootExtension(val rootProject: Project) {
    }
    var installationDir = gradleHome.resolve("nodejs")
    var cleanableStore = CleanableStore[installationDir.absolutePath]
    var download = true
    var nodeDownloadBaseUrl = "https://nodejs.org/dist"
@@ -64,7 +66,8 @@ open class NodeJsRootExtension(val rootProject: Project) {
            val platform = NodeJsPlatform.name
            val architecture = NodeJsPlatform.architecture
-            val nodeDir = installationDir.resolve("node-v$nodeVersion-$platform-$architecture")
+            val nodeDirName = "node-v$nodeVersion-$platform-$architecture"
            val nodeDir = cleanableStore[nodeDirName].use()
            val isWindows = NodeJsPlatform.name == NodeJsPlatform.WIN
            val nodeBinDir = if (isWindows) nodeDir else nodeDir.resolve("bin")
@@ -3,9 +3,9 @@ package org.jetbrains.kotlin.gradle.targets.js.nodejs
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.plugins.BasePlugin
 import org.gradle.api.tasks.Delete
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootExtension.Companion.EXTENSION_NAME
 import org.jetbrains.kotlin.gradle.targets.js.npm.tasks.KotlinNpmInstallTask
 import org.jetbrains.kotlin.gradle.tasks.CleanDataTask
 open class NodeJsRootPlugin : Plugin<Project> {
    override fun apply(project: Project): Unit = project.run {
@@ -15,7 +15,7 @@ open class NodeJsRootPlugin : Plugin<Project> {
            "NodeJsRootPlugin can be applied only to root project"
        }
-        this.extensions.create(EXTENSION_NAME, NodeJsRootExtension::class.java, this)
+        val settings = this.extensions.create(EXTENSION_NAME, NodeJsRootExtension::class.java, this)
        val setupTask = tasks.create(NodeJsSetupTask.NAME, NodeJsSetupTask::class.java) {
            it.group = TASKS_GROUP_NAME
@@ -27,6 +27,12 @@ open class NodeJsRootPlugin : Plugin<Project> {
            it.group = TASKS_GROUP_NAME
            it.description = "Find, download and link NPM dependencies and projects"
        }
        tasks.create("node" + CleanDataTask.NAME, CleanDataTask::class.java) {
            it.cleanableStore = settings.cleanableStore
            it.group = TASKS_GROUP_NAME
            it.description = "Clean unused local node version"
        }
    }
    companion object {
@@ -25,7 +25,8 @@ abstract class YarnBasics : NpmApi {
    ) {
        val nodeJs = NodeJsRootPlugin.apply(project)
        val nodeJsEnv = nodeJs.environment
-        val yarnEnv = YarnPlugin.apply(project).environment
+        val yarnPlugin = YarnPlugin.apply(project)
        val yarnEnv = yarnPlugin.environment
        project.execWithProgress(description) { exec ->
            exec.executable = nodeJsEnv.nodeExecutable
@@ -34,6 +35,7 @@ abstract class YarnBasics : NpmApi {
                    if (project.logger.isDebugEnabled) "--verbose" else ""
            exec.workingDir = dir
        }
    }
    protected fun yarnLockReadTransitiveDependencies(
@@ -8,6 +8,8 @@ package org.jetbrains.kotlin.gradle.targets.js.yarn
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootPlugin
 import org.jetbrains.kotlin.gradle.tasks.CleanDataTask
 import org.jetbrains.kotlin.gradle.tasks.internal.CleanableStore
 open class YarnPlugin : Plugin<Project> {
    override fun apply(project: Project): Unit = project.run {
@@ -17,10 +19,16 @@ open class YarnPlugin : Plugin<Project> {
        val nodeJs = NodeJsRootPlugin.apply(this)
-        this.extensions.create(YarnRootExtension.YARN, YarnRootExtension::class.java, this)
+        val yarnRootExtension = this.extensions.create(YarnRootExtension.YARN, YarnRootExtension::class.java, this)
        tasks.create(YarnSetupTask.NAME, YarnSetupTask::class.java) {
            it.dependsOn(nodeJs.nodeJsSetupTask)
        }
        tasks.create("yarn" + CleanDataTask.NAME, CleanDataTask::class.java) {
            it.cleanableStore = yarnRootExtension.cleanableStore
            it.description = "Clean unused local yarn version"
        }
    }
    companion object {
@@ -8,6 +8,7 @@ package org.jetbrains.kotlin.gradle.targets.js.yarn
 import org.gradle.api.Project
 import org.jetbrains.kotlin.gradle.logging.kotlinInfo
 import org.jetbrains.kotlin.gradle.targets.js.nodejs.NodeJsRootPlugin
 import org.jetbrains.kotlin.gradle.tasks.internal.CleanableStore
 open class YarnRootExtension(val project: Project) {
    init {
@@ -20,6 +21,9 @@ open class YarnRootExtension(val project: Project) {
    var installationDir = gradleHome.resolve("yarn")
    // TODO: Split configuration and execution phases to support changed installationDir
    val cleanableStore = CleanableStore[installationDir.path]
    var downloadBaseUrl = "https://github.com/yarnpkg/yarn/releases/download"
    var version = "1.21.1"
@@ -31,21 +35,20 @@ open class YarnRootExtension(val project: Project) {
    val useWorkspaces: Boolean
        get() = !disableWorkspaces
    // TODO: Split configuration and execution phases to support changed settings
    internal val environment = YarnEnv(
        downloadUrl = "$downloadBaseUrl/v$version/yarn-v$version.tar.gz",
        home = cleanableStore["yarn-v$version"].use()
    )
    internal fun executeSetup() {
        NodeJsRootPlugin.apply(project).executeSetup()
-        val env = environment
+        if (!environment.home.isDirectory) {
        if (!env.home.isDirectory) {
            yarnSetupTask.setup()
        }
    }
    internal val environment
        get() = YarnEnv(
            downloadUrl = "$downloadBaseUrl/v$version/yarn-v$version.tar.gz",
            home = installationDir.resolve("yarn-v$version")
        )
    companion object {
        const val YARN: String = "kotlinYarn"
@@ -0,0 +1,47 @@
 /*
 * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
 * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
 */
 package org.jetbrains.kotlin.gradle.tasks
 import org.gradle.api.DefaultTask
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskAction
 import org.jetbrains.kotlin.gradle.tasks.internal.CleanableStore
 import java.time.Duration
 import java.time.Instant
 /**
 * Task to clean all old loaded files based on a date of special access file.
 * The access file should be updated every time when loaded files are used
 */
 open class CleanDataTask : DefaultTask() {
    /**
     * Path to folder.
     * Use path instead of file to avoid file scanning for change check
     */
    @Input
    lateinit var cleanableStore: CleanableStore
    /**
     * Time to live in days
     */
    @Input
    var timeToLiveInDays: Long = 30
    @Suppress("unused")
    @TaskAction
    fun exec() {
        val expirationDate = Instant.now().minus(Duration.ofDays(timeToLiveInDays))
        cleanableStore.cleanDir(expirationDate)
    }
    companion object {
        const val NAME: String = "KotlinClean"
    }
 }
@@ -0,0 +1,39 @@
 /*
 * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
 * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
 */
 package org.jetbrains.kotlin.gradle.tasks
 import org.gradle.api.DefaultTask
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.TaskAction
 import org.jetbrains.kotlin.gradle.tasks.internal.CleanableStore
 import java.time.*
 /**
 * Task to clean all old loaded files based on a last modification date.
 * All registered store in {@link CleanableStore} would be cleaned
 */
 class CleanOldStoredDataTask : DefaultTask() {
    /**
     * Time to live in days
     */
    @Input
    val timeToLiveInDays: Long = 30
    @Suppress("unused")
    @TaskAction
    fun exec() {
        val expirationDate = Instant.now().minus(Duration.ofDays(timeToLiveInDays))
        CleanableStore.stores.forEach { (_, store) -> store.cleanDir(expirationDate) }
    }
    companion object {
        const val NAME: String = "clean store"
    }
 }
@@ -0,0 +1,35 @@
 /*
 * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
 * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
 */
 package org.jetbrains.kotlin.gradle.tasks.internal
 import org.jetbrains.kotlin.gradle.tasks.CleanDataTask
 import java.time.Instant
 /**
 * Store of tracked directories that can be cleaned with [CleanDataTask].
 * All directories that was not marked as used at least [CleanDataTask.timeToLiveInDays] days will be removed.
 *
 * To register store call `CleanableStore["/path/to/dir"]`.
 * Now you will be able to access files via `CleanableStore["/path/to/dir"]["file/name"].use()`
 * and it would update usage of th store.
 */
 interface CleanableStore {
    fun cleanDir(expirationDate: Instant)
    operator fun get(fileName: String): DownloadedFile
    fun markUsed()
    companion object {
        private val mutableStores = mutableMapOf<String, CleanableStore>()
        val stores: Map<String, CleanableStore>
            get() = mutableStores.toMap()
        operator fun get(path: String): CleanableStore =
            mutableStores.getOrPut(path) { CleanableStoreImpl(path) }
    }
 }
@@ -0,0 +1,36 @@
 /*
 * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
 * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
 */
 package org.jetbrains.kotlin.gradle.tasks.internal
 import org.gradle.util.GFileUtils
 import java.io.File
 import java.nio.file.Files
 import java.time.Instant
 internal class CleanableStoreImpl(dirPath: String) : CleanableStore {
    private val dir = File(dirPath)
    override fun get(fileName: String): DownloadedFile =
        DownloadedFile(this, dir.resolve(fileName))
    override fun markUsed() {
        if (dir.exists()) {
            GFileUtils.touchExisting(dir)
        }
    }
    override fun cleanDir(expirationDate: Instant) {
        fun modificationDate(file: File): Instant {
            return Files.getLastModifiedTime(file.toPath()).toInstant()
        }
        dir.listFiles()
            ?.filter { file ->
                modificationDate(file).isBefore(expirationDate)
            }
            ?.forEach { file -> file.deleteRecursively() }
    }
 }
@@ -0,0 +1,24 @@
 /*
 * Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
 * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
 */
 package org.jetbrains.kotlin.gradle.tasks.internal
 import java.io.File
 /**
 * Wrapper around [File] to be able to mark [store] storage as used by calling [use]`()`.
 */
 class DownloadedFile internal constructor(
    private val store: CleanableStore,
    private val file: File
 ) {
    fun use(): File {
        store.markUsed()
        return file
    }
    fun resolve(fileName: String): DownloadedFile =
        DownloadedFile(store, file.resolve(fileName))
 }