[Scripting] Add partial transitive resolution to Maven resolver

2022-09-09 19:46:23 +02:00
parent a78d43c1ec
commit 551a481dc5
4 changed files with 123 additions and 26 deletions
@@ -17,6 +17,7 @@ import kotlin.script.experimental.dependencies.ExternalDependenciesResolver
 import kotlin.script.experimental.dependencies.RepositoryCoordinates
 import kotlin.script.experimental.dependencies.impl.*
 import kotlin.script.experimental.dependencies.maven.impl.AetherResolveSession
+import kotlin.script.experimental.dependencies.maven.impl.ResolutionKind
 import kotlin.script.experimental.dependencies.maven.impl.mavenCentral

@Deprecated(
@@ -58,15 +59,20 @@ class MavenDependenciesResolver : ExternalDependenciesResolver {

        return try {
            val dependencyScopes = options.dependencyScopes ?: listOf(JavaScopes.COMPILE, JavaScopes.RUNTIME)
-            val transitive = options.transitive ?: true
+            val kind = when (options.partialResolution) {
+                true -> ResolutionKind.TRANSITIVE_PARTIAL
+                false, null -> when(options.transitive) {
+                    true, null -> ResolutionKind.TRANSITIVE
+                    false -> ResolutionKind.NON_TRANSITIVE
+                }
+            }
            val classifier = options.classifier
            val extension = options.extension
-            val deps = AetherResolveSession(
+            AetherResolveSession(
                null, remoteRepositories()
            ).resolve(
-                artifactId, dependencyScopes.joinToString(","), transitive, null, classifier, extension
+                artifactId, dependencyScopes.joinToString(","), kind, null, classifier, extension
            )
-            ResultWithDiagnostics.Success(deps.map { it.file })
        } catch (e: RepositoryException) {
            makeResolveFailureResult(e, sourceCodeLocation)
        }
@@ -39,10 +39,23 @@ import org.eclipse.aether.util.repository.AuthenticationBuilder
 import org.eclipse.aether.util.repository.DefaultMirrorSelector
 import org.eclipse.aether.util.repository.DefaultProxySelector
 import java.io.File
-import java.util.*
+import kotlin.script.experimental.api.ResultWithDiagnostics
+import kotlin.script.experimental.api.ScriptDiagnostic
+import kotlin.script.experimental.api.asSuccess

 val mavenCentral: RemoteRepository = RemoteRepository.Builder("maven central", "default", "https://repo.maven.apache.org/maven2/").build()

+internal enum class ResolutionKind {
+    NON_TRANSITIVE,
+    TRANSITIVE,
+
+    // Partial resolution is successful in case if dependency tree was built,
+    // but may return non-complete list of dependencies - i.e. while requesting sources, some libraries may lack sources artifacts.
+    // Resolution errors will be attached as reports.
+    // Also, might be slightly slower than usual transitive resolution.
+    TRANSITIVE_PARTIAL
+}
+
 internal class AetherResolveSession(
    localRepoDirectory: File? = null,
    remoteRepos: List<RemoteRepository> = listOf(mavenCentral)
@@ -133,22 +146,63 @@ internal class AetherResolveSession(
    fun resolve(
        root: Artifact,
        scope: String,
-        transitive: Boolean,
+        kind: ResolutionKind,
        filter: DependencyFilter?,
        classifier: String? = null,
        extension: String? = null,
-    ): List<Artifact> {
-        return if (transitive) resolveDependencies(root, scope, filter, classifier, extension)
-        else resolveArtifact(root)
+    ): ResultWithDiagnostics<List<File>> {
+        if (kind == ResolutionKind.NON_TRANSITIVE) return resolveArtifact(root).asSuccess()
+
+        val requests = resolveTree(root, scope, filter, classifier, extension)
+
+        @Suppress("KotlinConstantConditions")
+        return when (kind) {
+            ResolutionKind.TRANSITIVE -> resolveDependencies(requests) {
+                repositorySystem.resolveArtifacts(
+                    repositorySystemSession,
+                    requests
+                ).toFiles().asSuccess()
+            }
+
+            ResolutionKind.TRANSITIVE_PARTIAL -> resolveDependencies(requests) {
+                val reports = mutableListOf<ScriptDiagnostic>()
+                val results = mutableListOf<File>()
+                for (req in requests) {
+                    try {
+                        results.add(
+                            repositorySystem.resolveArtifact(
+                                repositorySystemSession,
+                                req
+                            ).artifact.file
+                        )
+                    } catch (e: ArtifactResolutionException) {
+                        reports.add(
+                            ScriptDiagnostic(
+                                ScriptDiagnostic.unspecifiedError,
+                                e.message.orEmpty(),
+                                ScriptDiagnostic.Severity.WARNING,
+                                exception = e
+                            )
+                        )
+                    }
+                }
+
+                ResultWithDiagnostics.Success(results, reports)
+            }
+
+            ResolutionKind.NON_TRANSITIVE -> {
+                error("This statement is not reachable")
+            }
+        }
    }

-    private fun resolveDependencies(
+    private fun resolveTree(
        root: Artifact,
        scope: String,
        filter: DependencyFilter?,
        classifier: String?,
        extension: String?,
-    ): List<Artifact> {
+    ): Collection<ArtifactRequest> {
        return fetch(
            request(Dependency(root, scope)),
            { req ->
@@ -163,8 +217,7 @@ internal class AetherResolveSession(
                    )
                )

-                val requests = requestsBuilder.requests
-                repositorySystem.resolveArtifacts(repositorySystemSession, requests)
+                requestsBuilder.requests
            },
            { req, ex ->
                DependencyCollectionException(
@@ -176,7 +229,25 @@ internal class AetherResolveSession(
        )
    }

-    private fun resolveArtifact(artifact: Artifact): List<Artifact> {
+    private fun Collection<ArtifactResult>.toFiles() = map { it.artifact.file }
+
+    private fun resolveDependencies(
+        requests: Collection<ArtifactRequest>,
+        resolveAction: (Collection<ArtifactRequest>) -> ResultWithDiagnostics<List<File>>
+    ): ResultWithDiagnostics<List<File>> {
+        return fetch(
+            requests,
+            resolveAction
+        ) { _, ex ->
+            DependencyCollectionException(
+                null,
+                ex.message,
+                ex
+            )
+        }
+    }
+
+    private fun resolveArtifact(artifact: Artifact): List<File> {
        val request = ArtifactRequest()
        request.artifact = artifact
        for (repo in remotes) {
@@ -187,7 +258,7 @@ internal class AetherResolveSession(
            request,
            { req -> listOf(repositorySystem.resolveArtifact(repositorySystemSession, req)) },
            { req, ex -> ArtifactResolutionException(listOf(ArtifactResult(req)), ex.message, IllegalArgumentException(ex)) }
-        )
+        ).toFiles()
    }

    private fun request(root: Dependency): CollectRequest {
@@ -199,25 +270,19 @@ internal class AetherResolveSession(
        return request
    }

-    private fun <RequestT> fetch(
+    private fun <RequestT, ResultT> fetch(
        request: RequestT,
-        fetchBody: (RequestT) -> Collection<ArtifactResult>,
+        fetchBody: (RequestT) -> ResultT,
        wrapException: (RequestT, Exception) -> Exception
-    ): List<Artifact> {
-        val deps: MutableList<Artifact> = LinkedList()
-        try {
-            var results: Collection<ArtifactResult>
+    ): ResultT {
+        return try {
            synchronized(this) {
-                results = fetchBody(request)
-            }
-            for (res in results) {
-                deps.add(res.artifact)
+                fetchBody(request)
            }
            // @checkstyle IllegalCatch (1 line)
        } catch (ex: Exception) {
            throw wrapException(request, ex)
        }
-        return deps
    }

    private fun getMirrorSelector(): DefaultMirrorSelector {
@@ -183,6 +183,24 @@ class MavenResolverTest : ResolversTestBase() {
        )
    }

+    @Ignore("ignored because spark is a very heavy dependency")
+    fun ignore_testPartialResolution() {
+        val resolver = MavenDependenciesResolver()
+        val options = buildOptions(
+            DependenciesResolverOptionsName.PARTIAL_RESOLUTION to "true",
+            DependenciesResolverOptionsName.CLASSIFIER to "sources",
+            DependenciesResolverOptionsName.EXTENSION to "jar",
+        )
+
+        val result = runBlocking {
+            resolver.resolve("org.jetbrains.kotlinx.spark:kotlin-spark-api_3.3.0_2.13:1.2.1", options)
+        }
+
+        result as ResultWithDiagnostics.Success
+        assertTrue(result.reports.isNotEmpty())
+        assertTrue(result.value.isNotEmpty())
+    }
+
    // Ignored - tests with custom repos often break the CI due to the caching issues
    // TODO: find a way to enable it back
    @Ignore
@@ -23,6 +23,7 @@ operator fun MutableMap<String, String>.set(key: DependenciesResolverOptionsName
 */
 enum class DependenciesResolverOptionsName(optionName: String? = null) {
    TRANSITIVE,
+    PARTIAL_RESOLUTION,
    SCOPE,
    USERNAME,
    PASSWORD,
@@ -37,6 +38,13 @@ enum class DependenciesResolverOptionsName(optionName: String? = null) {
 val ExternalDependenciesResolver.Options.transitive
    get() = flag(DependenciesResolverOptionsName.TRANSITIVE)

+/**
+ * Enables partial resolution of transitive dependencies.
+ * When this flag is enabled, resolver ignores [transitive] flag.
+ */
+val ExternalDependenciesResolver.Options.partialResolution
+    get() = flag(DependenciesResolverOptionsName.PARTIAL_RESOLUTION)
+
 val ExternalDependenciesResolver.Options.dependencyScopes
    get() = value(DependenciesResolverOptionsName.SCOPE)?.split(",")