Add new script dependency resolvers

Changes in new API for dependency resolvers (GenericDependenciesResolver) comparing to an old one (GenericResolver: - Add ability to fast check whether artifact or repository is suitable for current resolver to distinguish between unsuitable resolvers and resolution failures - Return all artifact resolution failures in ResultWithDiagnostics - Use single string for artifact coordinates - Add compound resolver that combines several resolvers - Merge Direct and FlatLib resolver into single FileSystemDependencyResolver - Mark resolve() method as suspend to indicate long operation - Add credentials support for maven resolver (https://youtrack.jetbrains.com/issue/KT-27701)
2019-09-27 14:01:02 +03:00
parent 81e2e119e2
commit c7e9a6c5d9
12 changed files with 535 additions and 0 deletions
@@ -535,6 +535,8 @@ tasks {
        dependsOn(":kotlin-scripting-common:test")
        dependsOn(":kotlin-scripting-jvm:test")
        dependsOn(":kotlin-scripting-jvm-host-test:test")
+        dependsOn(":kotlin-scripting-dependencies:test")
+        dependsOn(":kotlin-scripting-dependencies-maven:test")
        dependsOn(":kotlin-scripting-jsr223-test:test")
        // see comments on the task in kotlin-scripting-jvm-host-test
 //        dependsOn(":kotlin-scripting-jvm-host-test:embeddableTest")
@@ -0,0 +1,32 @@
+
+plugins {
+    kotlin("jvm")
+    id("jps-compatible")
+}
+
+jvmTarget = "1.6"
+
+dependencies {
+    compile(kotlinStdlib())
+    compile(project(":kotlin-scripting-dependencies"))
+    compile("org.jetbrains.kotlin:jcabi-aether:1.0-dev-3")
+    compile("org.sonatype.aether:aether-api:1.13.1")
+    testCompile("org.apache.maven:maven-settings-builder:3.0.5")
+    testCompile(projectTests(":kotlin-scripting-dependencies"))
+    testCompile(commonDep("junit"))
+}
+
+sourceSets {
+    "main" { projectDefault() }
+    "test" { projectDefault() }
+}
+
+tasks.withType<org.jetbrains.kotlin.gradle.dsl.KotlinCompile<*>> {
+    kotlinOptions.freeCompilerArgs += "-Xallow-kotlin-package"
+}
+
+publish()
+
+runtimeJar()
+sourcesJar()
+javadocJar()
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.script.experimental.dependencies.maven
+
+import com.jcabi.aether.Aether
+import org.sonatype.aether.repository.Authentication
+import org.sonatype.aether.repository.RemoteRepository
+import org.sonatype.aether.resolution.DependencyResolutionException
+import org.sonatype.aether.util.artifact.DefaultArtifact
+import org.sonatype.aether.util.artifact.JavaScopes
+import java.io.File
+import java.util.*
+import kotlin.script.experimental.api.ResultWithDiagnostics
+import kotlin.script.experimental.dependencies.ExternalDependenciesResolver
+import kotlin.script.experimental.dependencies.RepositoryCoordinates
+import kotlin.script.experimental.dependencies.impl.makeResolveFailureResult
+import kotlin.script.experimental.dependencies.impl.toRepositoryUrlOrNull
+
+val mavenCentral = RemoteRepository("maven-central", "default", "https://repo.maven.apache.org/maven2/")
+
+class MavenRepositoryCoordinates(
+    url: String,
+    val username: String?,
+    val password: String?,
+    val privateKeyFile: String?,
+    val passPhrase: String?
+) : RepositoryCoordinates(url)
+
+class MavenDependenciesResolver : ExternalDependenciesResolver {
+
+    override fun acceptsArtifact(artifactCoordinates: String): Boolean =
+        artifactCoordinates.toMavenArtifact() != null
+
+    override fun acceptsRepository(repositoryCoordinates: RepositoryCoordinates): Boolean {
+        return repositoryCoordinates.toRepositoryUrlOrNull() != null
+    }
+
+    // TODO: make robust
+    val localRepo = File(File(System.getProperty("user.home")!!, ".m2"), "repository")
+
+    val repos: ArrayList<RemoteRepository> = arrayListOf()
+
+    private fun remoteRepositories() = if (repos.isEmpty()) arrayListOf(mavenCentral) else repos
+
+    private fun allRepositories() = remoteRepositories().map { it.url!!.toString() } + localRepo.toString()
+
+    private fun String.toMavenArtifact(): DefaultArtifact? =
+        if (this.isNotBlank() && this.count { it == ':' } == 2) DefaultArtifact(this)
+        else null
+
+    override suspend fun resolve(artifactCoordinates: String): ResultWithDiagnostics<List<File>> {
+
+        val artifactId = artifactCoordinates.toMavenArtifact()!!
+
+        try {
+            val deps = Aether(remoteRepositories(), localRepo).resolve(artifactId, JavaScopes.RUNTIME)
+            if (deps != null)
+                return ResultWithDiagnostics.Success(deps.map { it.file })
+        } catch (e: DependencyResolutionException) {
+
+        }
+        return makeResolveFailureResult(allRepositories().map { "$it: $artifactId not found" })
+    }
+
+    private fun tryResolveEnvironmentVariable(str: String) =
+        if (str.startsWith("$")) System.getenv(str.substring(1)) ?: str
+        else str
+
+    override fun addRepository(repositoryCoordinates: RepositoryCoordinates) {
+        val url = repositoryCoordinates.toRepositoryUrlOrNull()
+            ?: throw IllegalArgumentException("Invalid Maven repository URL: ${repositoryCoordinates}")
+        val repo = RemoteRepository(
+            repositoryCoordinates.string,
+            "default",
+            url.toString()
+        )
+        if (repositoryCoordinates is MavenRepositoryCoordinates) {
+            val username = repositoryCoordinates.username?.let(::tryResolveEnvironmentVariable)
+            val password = repositoryCoordinates.password?.let(::tryResolveEnvironmentVariable)
+            repo.authentication = Authentication(username, password, repositoryCoordinates.privateKeyFile, repositoryCoordinates.passPhrase)
+        }
+        repos.add(repo)
+    }
+}
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.script.experimental.test
+
+import kotlinx.coroutines.runBlocking
+import org.junit.Assert
+import kotlin.contracts.ExperimentalContracts
+import kotlin.script.experimental.dependencies.maven.MavenDependenciesResolver
+import kotlin.script.experimental.api.ResultWithDiagnostics
+import kotlin.script.experimental.api.valueOrThrow
+
+@ExperimentalContracts
+class MavenResolverTest : ResolversTestBase() {
+
+    fun testResolve() {
+        val resolver = MavenDependenciesResolver()
+        val result = runBlocking { resolver.resolve("org.jetbrains.kotlin:kotlin-annotations-jvm:1.3.50") }
+        Assert.assertTrue(result is ResultWithDiagnostics.Success)
+        val files = result.valueOrThrow()
+        files.forEach { it.delete() }
+    }
+}
@@ -0,0 +1,29 @@
+
+plugins {
+    kotlin("jvm")
+    id("jps-compatible")
+}
+
+jvmTarget = "1.6"
+
+dependencies {
+    compile(kotlinStdlib())
+    compile(project(":kotlin-scripting-common"))
+    testCompile(commonDep("junit"))
+}
+
+sourceSets {
+    "main" { projectDefault() }
+    "test" { projectDefault() }
+}
+
+tasks.withType<org.jetbrains.kotlin.gradle.dsl.KotlinCompile<*>> {
+    kotlinOptions.freeCompilerArgs += "-Xallow-kotlin-package"
+}
+
+publish()
+
+runtimeJar()
+sourcesJar()
+javadocJar()
+testsJar()
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.script.experimental.dependencies
+
+import java.io.File
+import kotlin.script.experimental.api.ResultWithDiagnostics
+import kotlin.script.experimental.api.ScriptDiagnostic
+import kotlin.script.experimental.dependencies.impl.makeResolveFailureResult
+
+class CompoundDependenciesResolver(private val resolvers: List<ExternalDependenciesResolver>) : ExternalDependenciesResolver {
+
+    constructor(vararg resolvers: ExternalDependenciesResolver) : this(resolvers.toList())
+
+    override fun acceptsArtifact(artifactCoordinates: String): Boolean {
+        return resolvers.any { it.acceptsArtifact(artifactCoordinates) }
+    }
+
+    override fun acceptsRepository(repositoryCoordinates: RepositoryCoordinates): Boolean {
+        return resolvers.any { it.acceptsRepository(repositoryCoordinates) }
+    }
+
+    override fun addRepository(repositoryCoordinates: RepositoryCoordinates) {
+        if (resolvers.count { it.tryAddRepository(repositoryCoordinates) } == 0)
+            throw Exception("Failed to detect repository type: $repositoryCoordinates")
+    }
+
+    override suspend fun resolve(artifactCoordinates: String): ResultWithDiagnostics<List<File>> {
+
+        val reports = mutableListOf<ScriptDiagnostic>()
+
+        for (resolver in resolvers) {
+            if (resolver.acceptsArtifact(artifactCoordinates)) {
+                when (val resolveResult = resolver.resolve(artifactCoordinates)) {
+                    is ResultWithDiagnostics.Failure -> reports.addAll(resolveResult.reports)
+                    else -> return resolveResult
+                }
+            }
+        }
+        return if (reports.count() == 0) {
+            makeResolveFailureResult("No suitable dependency resolver found for artifact '$artifactCoordinates'")
+        } else {
+            ResultWithDiagnostics.Failure(reports)
+        }
+    }
+
+}
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.script.experimental.dependencies
+
+import java.io.File
+import kotlin.script.experimental.api.ResultWithDiagnostics
+import kotlin.script.experimental.api.valueOrNull
+
+open class RepositoryCoordinates(val string: String)
+
+interface ExternalDependenciesResolver {
+
+    fun acceptsRepository(repositoryCoordinates: RepositoryCoordinates): Boolean
+    fun acceptsArtifact(artifactCoordinates: String): Boolean
+
+    suspend fun resolve(artifactCoordinates: String): ResultWithDiagnostics<List<File>>
+    fun addRepository(repositoryCoordinates: RepositoryCoordinates)
+}
+
+fun ExternalDependenciesResolver.acceptsRepository(repositoryCoordinates: String): Boolean =
+    acceptsRepository(RepositoryCoordinates(repositoryCoordinates))
+
+fun ExternalDependenciesResolver.addRepository(repositoryCoordinates: String) = addRepository(RepositoryCoordinates(repositoryCoordinates))
+
+suspend fun ExternalDependenciesResolver.tryResolve(artifactCoordinates: String): List<File>? =
+    if (acceptsArtifact(artifactCoordinates)) resolve(artifactCoordinates).valueOrNull() else null
+
+fun ExternalDependenciesResolver.tryAddRepository(repositoryCoordinates: String) =
+    tryAddRepository(RepositoryCoordinates(repositoryCoordinates))
+
+fun ExternalDependenciesResolver.tryAddRepository(repositoryCoordinates: RepositoryCoordinates) =
+    if (acceptsRepository(repositoryCoordinates)) {
+        addRepository(repositoryCoordinates)
+        true
+    } else false
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.script.experimental.dependencies
+
+import java.io.File
+import kotlin.script.experimental.api.ResultWithDiagnostics
+import kotlin.script.experimental.dependencies.impl.makeResolveFailureResult
+import kotlin.script.experimental.dependencies.impl.toRepositoryUrlOrNull
+
+class FileSystemDependenciesResolver(vararg paths: File) : ExternalDependenciesResolver {
+
+    private fun String.toRepositoryFileOrNull(): File? =
+        File(this).takeIf { it.exists() && it.isDirectory }
+
+    private fun RepositoryCoordinates.toFilePath() =
+        (this.toRepositoryUrlOrNull()?.takeIf { it.protocol == "file" }?.path ?: string).toRepositoryFileOrNull()
+
+    override fun addRepository(repositoryCoordinates: RepositoryCoordinates) {
+        val repoDir = repositoryCoordinates.toFilePath()
+            ?: throw IllegalArgumentException("Invalid repository location: '${repositoryCoordinates}'")
+        localRepos.add(repoDir)
+    }
+
+    override suspend fun resolve(artifactCoordinates: String): ResultWithDiagnostics<List<File>> {
+        if (!acceptsArtifact(artifactCoordinates)) throw IllegalArgumentException("Path is invalid")
+
+        val messages = mutableListOf<String>()
+
+        for (repo in localRepos) {
+            // TODO: add coordinates and wildcard matching
+            val file = if (repo == null) File(artifactCoordinates) else File(repo, artifactCoordinates)
+            when {
+                !file.exists() -> messages.add("File '$file' does not exists")
+                !file.isFile && !file.isDirectory -> messages.add("Path '$file' is neither file nor directory")
+                else -> return ResultWithDiagnostics.Success(listOf(file))
+            }
+        }
+        return makeResolveFailureResult(messages)
+    }
+
+    override fun acceptsArtifact(artifactCoordinates: String) =
+        !artifactCoordinates.isBlank() // TODO: make check stronger, e.g. using NIO's Path
+
+    override fun acceptsRepository(repositoryCoordinates: RepositoryCoordinates): Boolean = repositoryCoordinates.toFilePath() != null
+
+    private val localRepos = arrayListOf<File?>(null)
+
+    init {
+        for (path in paths) {
+            require(path.exists() && path.isDirectory) { "Invalid flat lib directory repository path '$path'" }
+        }
+        localRepos.addAll(paths)
+    }
+}
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.script.experimental.dependencies.impl
+
+import java.net.MalformedURLException
+import java.net.URL
+import kotlin.script.experimental.dependencies.RepositoryCoordinates
+import kotlin.script.experimental.api.ResultWithDiagnostics
+import kotlin.script.experimental.api.ScriptDiagnostic
+
+fun makeResolveFailureResult(message: String) = makeResolveFailureResult(listOf(message))
+
+fun makeResolveFailureResult(messages: Iterable<String>) =
+    ResultWithDiagnostics.Failure(messages.map { ScriptDiagnostic(it, ScriptDiagnostic.Severity.WARNING) })
+
+fun RepositoryCoordinates.toRepositoryUrlOrNull(): URL? =
+    try {
+        URL(string)
+    } catch (_: MalformedURLException) {
+        null
+    }
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.script.experimental.test
+
+import java.io.File
+import kotlin.contracts.ExperimentalContracts
+import kotlin.script.experimental.dependencies.*
+import kotlin.script.experimental.api.ResultWithDiagnostics
+import kotlin.script.experimental.dependencies.impl.makeResolveFailureResult
+
+@ExperimentalContracts
+class ResolversTest : ResolversTestBase() {
+
+    private fun <T> withTempFile(body: (file: File) -> T): T {
+        createTempFile()
+        val file = createTempFile()
+        file.deleteOnExit()
+        try {
+            return body(file)
+        } finally {
+            file.delete()
+        }
+    }
+
+    private fun getNonExistingFile() = withTempFile { it }.also { assertFalse(it.exists()) }
+
+    fun testFileSystemResolver() {
+        withTempFile { file ->
+
+            val resolver = FileSystemDependenciesResolver()
+
+            resolver.assertNotResolve(1, file.name)
+            resolver.assertResolve(file, file.canonicalPath)
+
+            val dir = file.parent!!
+            resolver.assertAcceptsRepository(dir)
+            resolver.addRepository(dir)
+
+            resolver.assertResolve(file, file.name)
+        }
+    }
+
+    fun testFileSystemResolverFail() {
+        val file = getNonExistingFile()
+
+        val resolver = FileSystemDependenciesResolver()
+        resolver.assertAcceptsArtifact(file.path)
+        resolver.assertNotResolve(1, file.path)
+
+        resolver.addRepository(file.parent)
+
+        resolver.assertNotResolve(2, file.name)
+        resolver.assertNotResolve(2, file.absolutePath)
+
+        resolver.addRepository(file.parentFile.parent)
+        resolver.assertNotResolve(3, file.path)
+    }
+
+    fun testFileSystemAcceptsLinuxPath() {
+        val resolver = FileSystemDependenciesResolver()
+        resolver.assertAcceptsArtifact("/usr/local/bin/temp")
+    }
+
+    fun testFileSystemAcceptsWindowsPath() {
+        val resolver = FileSystemDependenciesResolver()
+        resolver.assertAcceptsArtifact("C:\\temp\\myfile")
+    }
+
+    fun testFileSystemNotAcceptsMavenPath() {
+        val resolver = FileSystemDependenciesResolver()
+        resolver.assertNotAcceptsArtifact("  ")
+    }
+
+    class TestDependenciesResolver(
+        val acceptsArt: (String) -> Boolean,
+        val doResolve: (String) -> File?,
+        val acceptsRepo: (String) -> Boolean,
+        val addRepo: (String) -> Unit
+    ) : ExternalDependenciesResolver {
+
+        override fun acceptsArtifact(artifactCoordinates: String): Boolean = acceptsArt(artifactCoordinates)
+
+        override suspend fun resolve(artifactCoordinates: String): ResultWithDiagnostics<List<File>> {
+            if (!acceptsArtifact(artifactCoordinates)) throw Exception("Path is invalid")
+            val file = doResolve(artifactCoordinates) ?: return makeResolveFailureResult("Failed to resolve '$artifactCoordinates'")
+            return ResultWithDiagnostics.Success(listOf(file))
+        }
+
+        override fun addRepository(repositoryCoordinates: RepositoryCoordinates) {
+            addRepo(repositoryCoordinates.string)
+        }
+
+        override fun acceptsRepository(repositoryCoordinates: RepositoryCoordinates): Boolean {
+            return acceptsRepo(repositoryCoordinates.string)
+        }
+    }
+
+    fun testCompoundResolver() {
+        val file1 = getNonExistingFile()
+        val file2 = getNonExistingFile()
+        val resolver1 = TestDependenciesResolver(acceptsArt = { it.startsWith("file") },
+                                                 doResolve = { if (it == "file1") file1 else null },
+                                                 acceptsRepo = { false },
+                                                 addRepo = {})
+
+        var prefix: String? = null
+        val resolver2 = TestDependenciesResolver(acceptsArt = { a -> prefix?.let { a.startsWith(it) } ?: false },
+                                                 doResolve = { if (it.contains(".")) file2 else null },
+                                                 acceptsRepo = { true },
+                                                 addRepo = { prefix = it })
+
+        val resolver = CompoundDependenciesResolver(resolver1, resolver2)
+
+        resolver.assertNotResolve(1, "abc")
+        resolver.assertNotResolve(1, "file2")
+        resolver.assertResolve(file1, "file1")
+        resolver.addRepository("fil")
+        resolver.assertNotResolve(2, "file2")
+        resolver.assertResolve(file2, "file2.txt")
+        resolver.assertResolve(file1, "file1")
+        resolver.assertResolve(file2, "fil1.txt")
+    }
+}
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.script.experimental.test
+
+import junit.framework.TestCase
+import kotlinx.coroutines.runBlocking
+import org.junit.Assert
+import java.io.File
+import kotlin.contracts.ExperimentalContracts
+import kotlin.contracts.contract
+import kotlin.script.experimental.dependencies.ExternalDependenciesResolver
+import kotlin.script.experimental.dependencies.acceptsRepository
+import kotlin.script.experimental.api.ResultWithDiagnostics
+import kotlin.script.experimental.api.valueOrThrow
+
+@ExperimentalContracts
+fun <T> assertIsFailure(r: ResultWithDiagnostics<T>) {
+    contract {
+        returns() implies (r is ResultWithDiagnostics.Failure)
+    }
+
+    Assert.assertTrue(r is ResultWithDiagnostics.Failure)
+}
+
+@ExperimentalContracts
+fun <T> assertIsSuccess(r: ResultWithDiagnostics<T>) {
+    contract {
+        returns() implies (r is ResultWithDiagnostics.Success<T>)
+    }
+
+    TestCase.assertTrue(r is ResultWithDiagnostics.Success<T>)
+}
+
+@ExperimentalContracts
+abstract class ResolversTestBase : TestCase() {
+    fun ExternalDependenciesResolver.assertNotResolve(expectedReportsCount: Int, path: String) {
+        val result = runBlocking { resolve(path) }
+        assertIsFailure(result)
+        assertEquals(expectedReportsCount, result.reports.count())
+    }
+
+    fun ExternalDependenciesResolver.assertAcceptsArtifact(path: String) = assertTrue(acceptsArtifact(path))
+
+    fun ExternalDependenciesResolver.assertNotAcceptsArtifact(path: String) = assertFalse(acceptsArtifact(path))
+
+    fun ExternalDependenciesResolver.assertAcceptsRepository(path: String) = assertTrue(acceptsRepository(path))
+
+    fun ExternalDependenciesResolver.assertResolve(expected: File, path: String) {
+
+        assertTrue(acceptsArtifact(path))
+
+        val result = runBlocking { resolve(path) }
+        assertIsSuccess(result)
+
+        val value = result.valueOrThrow()
+        assertEquals(1, value.count())
+        assertEquals(expected.canonicalPath, value.first().canonicalPath)
+    }
+}
@@ -211,6 +211,8 @@ include ":kotlin-build-common",
        ":kotlin-scripting-compiler-embeddable",
        ":kotlin-scripting-compiler-impl",
        ":kotlin-scripting-compiler-impl-embeddable",
+        ":kotlin-scripting-dependencies",
+        ":kotlin-scripting-dependencies-maven",
        ":kotlin-scripting-jsr223",
        ":kotlin-scripting-jsr223-test",
        ":kotlin-scripting-jsr223-embeddable",
@@ -398,6 +400,8 @@ project(':kotlin-scripting-jvm').projectDir = "$rootDir/libraries/scripting/jvm"
 project(':kotlin-scripting-jvm-host').projectDir = "$rootDir/libraries/scripting/jvm-host" as File
 project(':kotlin-scripting-jvm-host-test').projectDir = "$rootDir/libraries/scripting/jvm-host-test" as File
 project(':kotlin-scripting-jvm-host-embeddable').projectDir = "$rootDir/libraries/scripting/jvm-host-embeddable" as File
+project(':kotlin-scripting-dependencies').projectDir = "$rootDir/libraries/scripting/dependencies" as File
+project(':kotlin-scripting-dependencies-maven').projectDir = "$rootDir/libraries/scripting/dependencies-maven" as File
 project(':kotlin-scripting-jsr223').projectDir = "$rootDir/libraries/scripting/jsr223" as File
 project(':kotlin-scripting-jsr223-test').projectDir = "$rootDir/libraries/scripting/jsr223-test" as File
 project(':kotlin-scripting-jsr223-embeddable').projectDir = "$rootDir/libraries/scripting/jsr223-embeddable" as File