Scripting, KT-61727: Make it possible to resolve multiple artifacts at once

For MavenDependenciesResolver it improves performance drastically
depending on how strong transitive dependencies of the roots interfere

^KT-61727 Fixed

libraries/scripting/common/src/kotlin/script/experimental/api/errorHandling.kt

@@ -308,3 +308,37 @@ fun <R> ResultWithDiagnostics<R>.valueOrThrow(): R = valueOr {
     )
 }
 
+class IterableResultsCollector<T> {
+    private val diagnostics = mutableListOf<ScriptDiagnostic>()
+    private val values = mutableListOf<T>()
+
+    fun add(result: ResultWithDiagnostics<Iterable<T>>) {
+        diagnostics.addAll(result.reports)
+        if (result is ResultWithDiagnostics.Success) {
+            values.addAll(result.value)
+        }
+    }
+
+    fun addValue(value: T) {
+        values.add(value)
+    }
+
+    fun addDiagnostic(diagnostic: ScriptDiagnostic) {
+        diagnostics.add(diagnostic)
+    }
+
+    fun getResult(): ResultWithDiagnostics<List<T>> {
+        return if (values.isEmpty()) {
+            ResultWithDiagnostics.Failure(diagnostics)
+        } else {
+            ResultWithDiagnostics.Success(values, diagnostics)
+        }
+    }
+}
+
+fun <T> Iterable<ResultWithDiagnostics<Iterable<T>>>.asSuccessIfAny(): ResultWithDiagnostics<List<T>> {
+    return IterableResultsCollector<T>().run {
+        forEach { add(it) }
+        getResult()
+    }
+}

libraries/scripting/dependencies-maven/src/kotlin/script/experimental/dependencies/maven/MavenDependenciesResolver.kt

@@ -13,6 +13,7 @@ import org.eclipse.aether.util.artifact.JavaScopes
 import org.eclipse.aether.util.repository.AuthenticationBuilder
 import java.io.File
 import kotlin.script.experimental.api.*
+import kotlin.script.experimental.dependencies.ArtifactWithLocation
 import kotlin.script.experimental.dependencies.ExternalDependenciesResolver
 import kotlin.script.experimental.dependencies.RepositoryCoordinates
 import kotlin.script.experimental.dependencies.impl.*
@@ -50,12 +51,13 @@ class MavenDependenciesResolver : ExternalDependenciesResolver {
         else null
 
     override suspend fun resolve(
-        artifactCoordinates: String,
+        artifactsWithLocations: List<ArtifactWithLocation>,
-        options: ExternalDependenciesResolver.Options,
+        options: ExternalDependenciesResolver.Options
-        sourceCodeLocation: SourceCode.LocationWithId?
     ): ResultWithDiagnostics<List<File>> {
-
+        val firstArtifactWithLocation = artifactsWithLocations.firstOrNull() ?: return ResultWithDiagnostics.Success(emptyList())
-        val artifactId = artifactCoordinates.toMavenArtifact()!!
+        val artifactIds = artifactsWithLocations.map {
+            it.artifact.toMavenArtifact()!!
+        }
 
         return try {
             val dependencyScopes = options.dependencyScopes ?: listOf(JavaScopes.COMPILE, JavaScopes.RUNTIME)
@@ -71,10 +73,10 @@ class MavenDependenciesResolver : ExternalDependenciesResolver {
             AetherResolveSession(
                 null, remoteRepositories()
             ).resolve(
-                artifactId, dependencyScopes.joinToString(","), kind, null, classifier, extension
+                artifactIds, dependencyScopes.joinToString(","), kind, null, classifier, extension
             )
         } catch (e: RepositoryException) {
-            makeResolveFailureResult(e, sourceCodeLocation)
+            makeResolveFailureResult(e, firstArtifactWithLocation.sourceCodeLocation)
         }
     }
 

libraries/scripting/dependencies-maven/src/kotlin/script/experimental/dependencies/maven/impl/aether.kt

@@ -144,16 +144,16 @@ internal class AetherResolveSession(
     }
 
     fun resolve(
-        root: Artifact,
+        roots: List<Artifact>,
         scope: String,
         kind: ResolutionKind,
         filter: DependencyFilter?,
         classifier: String? = null,
         extension: String? = null,
     ): ResultWithDiagnostics<List<File>> {
-        if (kind == ResolutionKind.NON_TRANSITIVE) return resolveArtifact(root).asSuccess()
+        if (kind == ResolutionKind.NON_TRANSITIVE) return resolveArtifacts(roots).asSuccess()
 
-        val requests = resolveTree(root, scope, filter, classifier, extension)
+        val requests = resolveTree(roots, scope, filter, classifier, extension)
 
         @Suppress("KotlinConstantConditions")
         return when (kind) {
@@ -197,14 +197,14 @@ internal class AetherResolveSession(
     }
 
     private fun resolveTree(
-        root: Artifact,
+        roots: List<Artifact>,
         scope: String,
         filter: DependencyFilter?,
         classifier: String?,
         extension: String?,
     ): Collection<ArtifactRequest> {
         return fetch(
-            request(Dependency(root, scope)),
+            request(roots.map { root -> Dependency(root, scope) }),
             { req ->
                 val requestsBuilder = ArtifactRequestBuilder(classifier, extension)
                 val collectionResult = repositorySystem.collectDependencies(repositorySystemSession, req)
@@ -247,23 +247,21 @@ internal class AetherResolveSession(
         }
     }
 
-    private fun resolveArtifact(artifact: Artifact): List<File> {
+    private fun resolveArtifacts(artifacts: List<Artifact>): List<File> {
-        val request = ArtifactRequest()
+        val requests = artifacts.map { artifact ->
-        request.artifact = artifact
+            ArtifactRequest(artifact, remotes, "")
-        for (repo in remotes) {
-            request.addRepository(repo)
         }
 
         return fetch(
-            request,
+            requests,
-            { req -> listOf(repositorySystem.resolveArtifact(repositorySystemSession, req)) },
+            { reqs -> listOf(repositorySystem.resolveArtifacts(repositorySystemSession, reqs)) },
-            { req, ex -> ArtifactResolutionException(listOf(ArtifactResult(req)), ex.message, IllegalArgumentException(ex)) }
+            { reqs, ex -> ArtifactResolutionException(reqs.map { req -> ArtifactResult(req) }, ex.message, IllegalArgumentException(ex)) }
-        ).toFiles()
+        ).flatMap { it.toFiles() }
     }
 
-    private fun request(root: Dependency): CollectRequest {
+    private fun request(roots: List<Dependency>): CollectRequest {
         val request = CollectRequest()
-        request.root = root
+        request.dependencies = roots
         for (repo in remotes) {
             request.addRepository(repo)
         }

libraries/scripting/dependencies-maven/test/kotlin/script/experimental/test/MavenResolverTest.kt

@@ -9,6 +9,7 @@ import kotlinx.coroutines.runBlocking
 import org.junit.Assert
 import org.junit.Ignore
 import java.io.File
+import java.util.*
 import kotlin.contracts.ExperimentalContracts
 import kotlin.contracts.InvocationKind
 import kotlin.contracts.contract
@@ -22,6 +23,7 @@ import kotlin.script.experimental.dependencies.impl.makeExternalDependenciesReso
 import kotlin.script.experimental.dependencies.impl.set
 import kotlin.script.experimental.dependencies.maven.MavenDependenciesResolver
 import kotlin.script.experimental.dependencies.maven.impl.createMavenSettings
+import kotlin.system.measureTimeMillis
 
 @ExperimentalContracts
 class MavenResolverTest : ResolversTestBase() {
@@ -183,6 +185,41 @@ class MavenResolverTest : ResolversTestBase() {
         )
     }
 
+    fun testMultipleDependencies() {
+        val resolver = MavenDependenciesResolver()
+        val sourceOptions = buildOptions(
+            DependenciesResolverOptionsName.PARTIAL_RESOLUTION to "true",
+            DependenciesResolverOptionsName.CLASSIFIER to "sources",
+            DependenciesResolverOptionsName.EXTENSION to "jar",
+        )
+        val multipleDependencies = listOf(
+            "commons-io:commons-io:2.11.0",
+            "org.jetbrains.kotlin:kotlin-reflect:1.8.20",
+            "org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.8.20",
+        ).map { ArtifactWithLocation(it, null) }
+
+        val result = TreeSet<String>()
+        fun addToResult(resultFiles: ResultWithDiagnostics<List<File>>) {
+            if (resultFiles is ResultWithDiagnostics.Success) {
+                for (file in resultFiles.value) {
+                    result.add(file.absolutePath)
+                }
+            }
+        }
+
+        val timeMs = measureTimeMillis {
+            runBlocking {
+                addToResult(resolver.resolve(multipleDependencies))
+                addToResult(resolver.resolve(multipleDependencies, sourceOptions))
+            }
+        }
+        println("Test time: $timeMs ms")
+        println("Deps size: ${result.size}")
+        println(result.joinToString("\n"))
+
+        assertEquals(14, result.size)
+    }
+
     @Ignore("ignored because spark is a very heavy dependency")
     fun ignore_testPartialResolution() {
         val resolver = MavenDependenciesResolver()

libraries/scripting/dependencies/src/kotlin/script/experimental/dependencies/CompoundDependenciesResolver.kt

@@ -6,10 +6,7 @@
 package kotlin.script.experimental.dependencies
 
 import java.io.File
-import kotlin.script.experimental.api.ResultWithDiagnostics
+import kotlin.script.experimental.api.*
-import kotlin.script.experimental.api.ScriptDiagnostic
-import kotlin.script.experimental.api.SourceCode
-import kotlin.script.experimental.api.asSuccess
 import kotlin.script.experimental.dependencies.impl.makeResolveFailureResult
 
 class CompoundDependenciesResolver(private val resolvers: List<ExternalDependenciesResolver>) : ExternalDependenciesResolver {
@@ -57,25 +54,57 @@ class CompoundDependenciesResolver(private val resolvers: List<ExternalDependenc
     }
 
     override suspend fun resolve(
-        artifactCoordinates: String,
+        artifactsWithLocations: List<ArtifactWithLocation>,
-        options: ExternalDependenciesResolver.Options,
+        options: ExternalDependenciesResolver.Options
-        sourceCodeLocation: SourceCode.LocationWithId?
     ): ResultWithDiagnostics<List<File>> {
-        val reports = mutableListOf<ScriptDiagnostic>()
+        val resultsCollector = IterableResultsCollector<File>()
 
-        for (resolver in resolvers) {
+        val artifactToResolverIndex = mutableMapOf<ArtifactWithLocation, Int>().apply {
-            if (resolver.acceptsArtifact(artifactCoordinates)) {
+            for (artifactWithLocation in artifactsWithLocations) {
-                when (val resolveResult = resolver.resolve(artifactCoordinates, options, sourceCodeLocation)) {
+                put(artifactWithLocation, -1)
-                    is ResultWithDiagnostics.Failure -> reports.addAll(resolveResult.reports)
+            }
-                    else -> return resolveResult
+        }
+
+        while (artifactToResolverIndex.isNotEmpty()) {
+            val resolverGroups = mutableMapOf<Int, MutableList<ArtifactWithLocation>>()
+
+            for ((artifactWithLocation, resolverIndex) in artifactToResolverIndex) {
+                val (artifact, sourceCodeLocation) = artifactWithLocation
+
+                var currentIndex = resolverIndex + 1
+                while (currentIndex < resolvers.size) {
+                    if (resolvers[currentIndex].acceptsArtifact(artifact)) break
+                    ++currentIndex
+                }
+                if (currentIndex == resolvers.size) {
+                    if (resolverIndex == -1) {
+                        // We add this diagnostic only if there were no resolution attempts made
+                        resultsCollector.addDiagnostic(
+                            "No suitable dependency resolver found for artifact '$artifact'"
+                                .asErrorDiagnostics(locationWithId = sourceCodeLocation)
+                        )
+                    }
+                } else {
+                    resolverGroups
+                        .getOrPut(currentIndex) { mutableListOf() }
+                        .add(artifactWithLocation)
+                }
+            }
+
+            artifactToResolverIndex.clear()
+            for ((resolverIndex, artifacts) in resolverGroups) {
+                val resolver = resolvers[resolverIndex]
+                val resolveResult = resolver.resolve(artifacts, options)
+                resultsCollector.add(resolveResult)
+                if (resolveResult.reports.isNotEmpty()) {
+                    for (artifact in artifacts) {
+                        artifactToResolverIndex[artifact] = resolverIndex
+                    }
                 }
             }
         }
-        return if (reports.count() == 0) {
+
-            makeResolveFailureResult("No suitable dependency resolver found for artifact '$artifactCoordinates'", sourceCodeLocation)
+        return resultsCollector.getResult()
-        } else {
-            ResultWithDiagnostics.Failure(reports)
-        }
     }
 
-}
+}

libraries/scripting/dependencies/src/kotlin/script/experimental/dependencies/ExternalDependenciesResolver.kt

@@ -11,6 +11,8 @@ import kotlin.script.experimental.dependencies.ExternalDependenciesResolver.Opti
 
 open class RepositoryCoordinates(val string: String)
 
+data class ArtifactWithLocation(val artifact: String, val sourceCodeLocation: SourceCode.LocationWithId?)
+
 interface ExternalDependenciesResolver {
     interface Options {
         object Empty : Options {
@@ -25,11 +27,18 @@ interface ExternalDependenciesResolver {
     fun acceptsRepository(repositoryCoordinates: RepositoryCoordinates): Boolean
     fun acceptsArtifact(artifactCoordinates: String): Boolean
 
+    // Override one of the following methods
     suspend fun resolve(
         artifactCoordinates: String,
         options: Options = Options.Empty,
         sourceCodeLocation: SourceCode.LocationWithId? = null
-    ): ResultWithDiagnostics<List<File>>
+    ): ResultWithDiagnostics<List<File>> = resolve(listOf(ArtifactWithLocation(artifactCoordinates, sourceCodeLocation)), options)
+
+    suspend fun resolve(
+        artifactsWithLocations: List<ArtifactWithLocation>,
+        options: Options = Options.Empty,
+    ): ResultWithDiagnostics<List<File>> =
+        artifactsWithLocations.map { (artifact, location) -> resolve(artifact, options, location) }.asSuccessIfAny()
 
     fun addRepository(
         repositoryCoordinates: RepositoryCoordinates,
@@ -46,4 +55,3 @@ fun ExternalDependenciesResolver.addRepository(
     options: Options = Options.Empty,
     sourceCodeLocation: SourceCode.LocationWithId? = null
 ) = addRepository(RepositoryCoordinates(repositoryCoordinates), options, sourceCodeLocation)
-

libraries/scripting/dependencies/test/kotlin/script/experimental/test/ResolversTestBase.kt

@@ -39,7 +39,7 @@ abstract class ResolversTestBase : TestCase() {
     fun ExternalDependenciesResolver.assertNotResolve(expectedReportsCount: Int, path: String) {
         val result = runBlocking { resolve(path) }
         assertIsFailure(result)
-        assertEquals(expectedReportsCount, result.reports.count())
+        assertEquals("Actual reports:\n${result.reports.joinToString("\n")}", expectedReportsCount, result.reports.count())
     }
 
     fun ExternalDependenciesResolver.assertAcceptsArtifact(path: String) = assertTrue(acceptsArtifact(path))