azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scripting, KT-61727: Make it possible to resolve multiple artifacts at once

Browse Source 
For MavenDependenciesResolver it improves performance drastically
depending on how strong transitive dependencies of the roots interfere

^KT-61727 Fixed
This commit is contained in:
Ilya Muradyan
2023-09-01 01:30:33 +02:00
committed by Space Team
parent 0d628c9e59
commit 841cc8eb43
7 changed files with 153 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/scripting/common/src/kotlin/script/experimental/api/errorHandling.kt
+34
View File
@@ -308,3 +308,37 @@ fun <R> ResultWithDiagnostics<R>.valueOrThrow(): R = valueOr {
)
}
class IterableResultsCollector<T> {
private val diagnostics = mutableListOf<ScriptDiagnostic>()
private val values = mutableListOf<T>()
fun add(result: ResultWithDiagnostics<Iterable<T>>) {
diagnostics.addAll(result.reports)
if (result is ResultWithDiagnostics.Success) {
values.addAll(result.value)
}
}
fun addValue(value: T) {
values.add(value)
}
fun addDiagnostic(diagnostic: ScriptDiagnostic) {
diagnostics.add(diagnostic)
}
fun getResult(): ResultWithDiagnostics<List<T>> {
return if (values.isEmpty()) {
ResultWithDiagnostics.Failure(diagnostics)
} else {
ResultWithDiagnostics.Success(values, diagnostics)
}
}
}
fun <T> Iterable<ResultWithDiagnostics<Iterable<T>>>.asSuccessIfAny(): ResultWithDiagnostics<List<T>> {
return IterableResultsCollector<T>().run {
forEach { add(it) }
getResult()
}
}
libraries/scripting/dependencies-maven/src/kotlin/script/experimental/dependencies/maven/MavenDependenciesResolver.kt
+9 -7
View File
@@ -13,6 +13,7 @@ import org.eclipse.aether.util.artifact.JavaScopes
import org.eclipse.aether.util.repository.AuthenticationBuilder
import java.io.File
import kotlin.script.experimental.api.*
import kotlin.script.experimental.dependencies.ArtifactWithLocation
import kotlin.script.experimental.dependencies.ExternalDependenciesResolver
import kotlin.script.experimental.dependencies.RepositoryCoordinates
import kotlin.script.experimental.dependencies.impl.*
@@ -50,12 +51,13 @@ class MavenDependenciesResolver : ExternalDependenciesResolver {
else null
override suspend fun resolve(
artifactCoordinates: String,
options: ExternalDependenciesResolver.Options,
sourceCodeLocation: SourceCode.LocationWithId?
artifactsWithLocations: List<ArtifactWithLocation>,
options: ExternalDependenciesResolver.Options
): ResultWithDiagnostics<List<File>> {
val artifactId = artifactCoordinates.toMavenArtifact()!!
val firstArtifactWithLocation = artifactsWithLocations.firstOrNull() ?: return ResultWithDiagnostics.Success(emptyList())
val artifactIds = artifactsWithLocations.map {
it.artifact.toMavenArtifact()!!
}
return try {
val dependencyScopes = options.dependencyScopes ?: listOf(JavaScopes.COMPILE, JavaScopes.RUNTIME)
@@ -71,10 +73,10 @@ class MavenDependenciesResolver : ExternalDependenciesResolver {
AetherResolveSession(
null, remoteRepositories()
).resolve(
artifactId, dependencyScopes.joinToString(","), kind, null, classifier, extension
artifactIds, dependencyScopes.joinToString(","), kind, null, classifier, extension
)
} catch (e: RepositoryException) {
makeResolveFailureResult(e, sourceCodeLocation)
makeResolveFailureResult(e, firstArtifactWithLocation.sourceCodeLocation)
}
}
libraries/scripting/dependencies-maven/src/kotlin/script/experimental/dependencies/maven/impl/aether.kt
+14 -16
View File
@@ -144,16 +144,16 @@ internal class AetherResolveSession(
}
fun resolve(
root: Artifact,
roots: List<Artifact>,
scope: String,
kind: ResolutionKind,
filter: DependencyFilter?,
classifier: String? = null,
extension: String? = null,
): ResultWithDiagnostics<List<File>> {
if (kind == ResolutionKind.NON_TRANSITIVE) return resolveArtifact(root).asSuccess()
if (kind == ResolutionKind.NON_TRANSITIVE) return resolveArtifacts(roots).asSuccess()
val requests = resolveTree(root, scope, filter, classifier, extension)
val requests = resolveTree(roots, scope, filter, classifier, extension)
@Suppress("KotlinConstantConditions")
return when (kind) {
@@ -197,14 +197,14 @@ internal class AetherResolveSession(
}
private fun resolveTree(
root: Artifact,
roots: List<Artifact>,
scope: String,
filter: DependencyFilter?,
classifier: String?,
extension: String?,
): Collection<ArtifactRequest> {
return fetch(
request(Dependency(root, scope)),
request(roots.map { root -> Dependency(root, scope) }),
{ req ->
val requestsBuilder = ArtifactRequestBuilder(classifier, extension)
val collectionResult = repositorySystem.collectDependencies(repositorySystemSession, req)
@@ -247,23 +247,21 @@ internal class AetherResolveSession(
}
}
private fun resolveArtifact(artifact: Artifact): List<File> {
val request = ArtifactRequest()
request.artifact = artifact
for (repo in remotes) {
request.addRepository(repo)
private fun resolveArtifacts(artifacts: List<Artifact>): List<File> {
val requests = artifacts.map { artifact ->
ArtifactRequest(artifact, remotes, "")
}
return fetch(
request,
{ req -> listOf(repositorySystem.resolveArtifact(repositorySystemSession, req)) },
{ req, ex -> ArtifactResolutionException(listOf(ArtifactResult(req)), ex.message, IllegalArgumentException(ex)) }
).toFiles()
requests,
{ reqs -> listOf(repositorySystem.resolveArtifacts(repositorySystemSession, reqs)) },
{ reqs, ex -> ArtifactResolutionException(reqs.map { req -> ArtifactResult(req) }, ex.message, IllegalArgumentException(ex)) }
).flatMap { it.toFiles() }
}
private fun request(root: Dependency): CollectRequest {
private fun request(roots: List<Dependency>): CollectRequest {
val request = CollectRequest()
request.root = root
request.dependencies = roots
for (repo in remotes) {
request.addRepository(repo)
}
libraries/scripting/dependencies-maven/test/kotlin/script/experimental/test/MavenResolverTest.kt
+37
View File
@@ -9,6 +9,7 @@ import kotlinx.coroutines.runBlocking
import org.junit.Assert
import org.junit.Ignore
import java.io.File
import java.util.*
import kotlin.contracts.ExperimentalContracts
import kotlin.contracts.InvocationKind
import kotlin.contracts.contract
@@ -22,6 +23,7 @@ import kotlin.script.experimental.dependencies.impl.makeExternalDependenciesReso
import kotlin.script.experimental.dependencies.impl.set
import kotlin.script.experimental.dependencies.maven.MavenDependenciesResolver
import kotlin.script.experimental.dependencies.maven.impl.createMavenSettings
import kotlin.system.measureTimeMillis
@ExperimentalContracts
class MavenResolverTest : ResolversTestBase() {
@@ -183,6 +185,41 @@ class MavenResolverTest : ResolversTestBase() {
)
}
fun testMultipleDependencies() {
val resolver = MavenDependenciesResolver()
val sourceOptions = buildOptions(
DependenciesResolverOptionsName.PARTIAL_RESOLUTION to "true",
DependenciesResolverOptionsName.CLASSIFIER to "sources",
DependenciesResolverOptionsName.EXTENSION to "jar",
)
val multipleDependencies = listOf(
"commons-io:commons-io:2.11.0",
"org.jetbrains.kotlin:kotlin-reflect:1.8.20",
"org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.8.20",
).map { ArtifactWithLocation(it, null) }
val result = TreeSet<String>()
fun addToResult(resultFiles: ResultWithDiagnostics<List<File>>) {
if (resultFiles is ResultWithDiagnostics.Success) {
for (file in resultFiles.value) {
result.add(file.absolutePath)
}
}
}
val timeMs = measureTimeMillis {
runBlocking {
addToResult(resolver.resolve(multipleDependencies))
addToResult(resolver.resolve(multipleDependencies, sourceOptions))
}
}
println("Test time: $timeMs ms")
println("Deps size: ${result.size}")
println(result.joinToString("\n"))
assertEquals(14, result.size)
}
@Ignore("ignored because spark is a very heavy dependency")
fun ignore_testPartialResolution() {
val resolver = MavenDependenciesResolver()
libraries/scripting/dependencies/src/kotlin/script/experimental/dependencies/CompoundDependenciesResolver.kt
+48 -19
View File
@@ -6,10 +6,7 @@
package kotlin.script.experimental.dependencies
import java.io.File
import kotlin.script.experimental.api.ResultWithDiagnostics
import kotlin.script.experimental.api.ScriptDiagnostic
import kotlin.script.experimental.api.SourceCode
import kotlin.script.experimental.api.asSuccess
import kotlin.script.experimental.api.*
import kotlin.script.experimental.dependencies.impl.makeResolveFailureResult
class CompoundDependenciesResolver(private val resolvers: List<ExternalDependenciesResolver>) : ExternalDependenciesResolver {
@@ -57,25 +54,57 @@ class CompoundDependenciesResolver(private val resolvers: List<ExternalDependenc
}
override suspend fun resolve(
artifactCoordinates: String,
options: ExternalDependenciesResolver.Options,
sourceCodeLocation: SourceCode.LocationWithId?
artifactsWithLocations: List<ArtifactWithLocation>,
options: ExternalDependenciesResolver.Options
): ResultWithDiagnostics<List<File>> {
val reports = mutableListOf<ScriptDiagnostic>()
val resultsCollector = IterableResultsCollector<File>()
for (resolver in resolvers) {
if (resolver.acceptsArtifact(artifactCoordinates)) {
when (val resolveResult = resolver.resolve(artifactCoordinates, options, sourceCodeLocation)) {
is ResultWithDiagnostics.Failure -> reports.addAll(resolveResult.reports)
else -> return resolveResult
val artifactToResolverIndex = mutableMapOf<ArtifactWithLocation, Int>().apply {
for (artifactWithLocation in artifactsWithLocations) {
put(artifactWithLocation, -1)
}
}
while (artifactToResolverIndex.isNotEmpty()) {
val resolverGroups = mutableMapOf<Int, MutableList<ArtifactWithLocation>>()
for ((artifactWithLocation, resolverIndex) in artifactToResolverIndex) {
val (artifact, sourceCodeLocation) = artifactWithLocation
var currentIndex = resolverIndex + 1
while (currentIndex < resolvers.size) {
if (resolvers[currentIndex].acceptsArtifact(artifact)) break
++currentIndex
}
if (currentIndex == resolvers.size) {
if (resolverIndex == -1) {
// We add this diagnostic only if there were no resolution attempts made
resultsCollector.addDiagnostic(
"No suitable dependency resolver found for artifact '$artifact'"
.asErrorDiagnostics(locationWithId = sourceCodeLocation)
)
}
} else {
resolverGroups
.getOrPut(currentIndex) { mutableListOf() }
.add(artifactWithLocation)
}
}
artifactToResolverIndex.clear()
for ((resolverIndex, artifacts) in resolverGroups) {
val resolver = resolvers[resolverIndex]
val resolveResult = resolver.resolve(artifacts, options)
resultsCollector.add(resolveResult)
if (resolveResult.reports.isNotEmpty()) {
for (artifact in artifacts) {
artifactToResolverIndex[artifact] = resolverIndex
}
}
}
}
return if (reports.count() == 0) {
makeResolveFailureResult("No suitable dependency resolver found for artifact '$artifactCoordinates'", sourceCodeLocation)
} else {
ResultWithDiagnostics.Failure(reports)
}
return resultsCollector.getResult()
}
}
}
libraries/scripting/dependencies/src/kotlin/script/experimental/dependencies/ExternalDependenciesResolver.kt
+10 -2
View File
@@ -11,6 +11,8 @@ import kotlin.script.experimental.dependencies.ExternalDependenciesResolver.Opti
open class RepositoryCoordinates(val string: String)
data class ArtifactWithLocation(val artifact: String, val sourceCodeLocation: SourceCode.LocationWithId?)
interface ExternalDependenciesResolver {
interface Options {
object Empty : Options {
@@ -25,11 +27,18 @@ interface ExternalDependenciesResolver {
fun acceptsRepository(repositoryCoordinates: RepositoryCoordinates): Boolean
fun acceptsArtifact(artifactCoordinates: String): Boolean
// Override one of the following methods
suspend fun resolve(
artifactCoordinates: String,
options: Options = Options.Empty,
sourceCodeLocation: SourceCode.LocationWithId? = null
): ResultWithDiagnostics<List<File>>
): ResultWithDiagnostics<List<File>> = resolve(listOf(ArtifactWithLocation(artifactCoordinates, sourceCodeLocation)), options)
suspend fun resolve(
artifactsWithLocations: List<ArtifactWithLocation>,
options: Options = Options.Empty,
): ResultWithDiagnostics<List<File>> =
artifactsWithLocations.map { (artifact, location) -> resolve(artifact, options, location) }.asSuccessIfAny()
fun addRepository(
repositoryCoordinates: RepositoryCoordinates,
@@ -46,4 +55,3 @@ fun ExternalDependenciesResolver.addRepository(
options: Options = Options.Empty,
sourceCodeLocation: SourceCode.LocationWithId? = null
) = addRepository(RepositoryCoordinates(repositoryCoordinates), options, sourceCodeLocation)
libraries/scripting/dependencies/test/kotlin/script/experimental/test/ResolversTestBase.kt
+1 -1
View File
@@ -39,7 +39,7 @@ abstract class ResolversTestBase : TestCase() {
fun ExternalDependenciesResolver.assertNotResolve(expectedReportsCount: Int, path: String) {
val result = runBlocking { resolve(path) }
assertIsFailure(result)
assertEquals(expectedReportsCount, result.reports.count())
assertEquals("Actual reports:\n${result.reports.joinToString("\n")}", expectedReportsCount, result.reports.count())
}
fun ExternalDependenciesResolver.assertAcceptsArtifact(path: String) = assertTrue(acceptsArtifact(path))