From 61c1312f1affd4f2c995412b1fa18b40513394e9 Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Wed, 24 Apr 2019 12:10:28 +0200 Subject: [PATCH] Add another method for fetching classpath from classloader, refactor --- ...DefinitionsFromClasspathDiscoverySource.kt | 219 +++++++++++------- 1 file changed, 131 insertions(+), 88 deletions(-) diff --git a/plugins/scripting/scripting-compiler-impl/src/org/jetbrains/kotlin/scripting/definitions/ScriptiDefinitionsFromClasspathDiscoverySource.kt b/plugins/scripting/scripting-compiler-impl/src/org/jetbrains/kotlin/scripting/definitions/ScriptiDefinitionsFromClasspathDiscoverySource.kt index 5c136461e46..ed2dff544a7 100644 --- a/plugins/scripting/scripting-compiler-impl/src/org/jetbrains/kotlin/scripting/definitions/ScriptiDefinitionsFromClasspathDiscoverySource.kt +++ b/plugins/scripting/scripting-compiler-impl/src/org/jetbrains/kotlin/scripting/definitions/ScriptiDefinitionsFromClasspathDiscoverySource.kt @@ -11,6 +11,7 @@ import org.jetbrains.kotlin.scripting.resolve.KotlinScriptDefinitionFromAnnotate import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull import java.io.File import java.io.IOException +import java.lang.IllegalArgumentException import java.net.URLClassLoader import java.util.jar.JarFile import kotlin.script.experimental.annotations.KotlinScript @@ -38,90 +39,123 @@ class ScriptDefinitionsFromClasspathDiscoverySource( } } -fun discoverScriptTemplatesInClasspath( - classpath: List, - baseClassLoader: ClassLoader, +private const val MANIFEST_RESOURCE_NAME = "/META-INF/MANIFEST.MF" + +fun discoverScriptTemplatesInClassLoader( + classLoader: ClassLoader, scriptResolverEnv: Map, messageCollector: MessageCollector -): Sequence = sequence { - // TODO: try to find a way to reduce classpath (and classloader) to minimal one needed to load script definition and its dependencies - val loader = LazyClasspathWithClassLoader(baseClassLoader) { classpath } - - // for jar files the definition class is expected in the same jar as the discovery file - // in case of directories, the class output may come separate from the resources, so some candidates should be deffered and processed later - val defferedDirDependencies = ArrayList() - val defferedDefinitionCandidates = ArrayList() - for (dep in classpath) { +): Sequence { + val classpath = classLoader.getResources(MANIFEST_RESOURCE_NAME).asSequence().mapNotNull { try { - when { - dep.isFile && dep.extension == "jar" -> { // checking for extension is the compiler current behaviour, so the same logic is implemented here - JarFile(dep).use { jar -> - if (jar.getJarEntry(SCRIPT_DEFINITION_MARKERS_PATH) != null) { - val definitionNames = jar.entries().asSequence().mapNotNull { - if (it.isDirectory || !it.name.startsWith(SCRIPT_DEFINITION_MARKERS_PATH)) null - else it.name.removePrefix(SCRIPT_DEFINITION_MARKERS_PATH).removeSuffix( - SCRIPT_DEFINITION_MARKERS_EXTENSION_WITH_DOT - ) - }.toList() - val (loadedDefinitions, notFoundClasses) = - definitionNames.partitionLoadJarDefinitions(jar, loader, scriptResolverEnv, messageCollector) - if (notFoundClasses.isNotEmpty()) { - messageCollector.report( - CompilerMessageSeverity.STRONG_WARNING, - "Configure scripting: unable to find script definitions [${notFoundClasses.joinToString(", ")}]" - ) + File(it.toURI()).takeIf(File::exists) + } catch (_: IllegalArgumentException) { + null + } + } + val classpathWithLoader = SimpleClasspathWithClassLoader(classpath.toList(), classLoader) + return scriptTemplatesDiscoverySequence(classpathWithLoader, scriptResolverEnv, messageCollector) +} + +fun discoverScriptTemplatesInClasspath( + classpath: List, + baseClassLoader: ClassLoader?, + scriptResolverEnv: Map, + messageCollector: MessageCollector +): Sequence { + // TODO: try to find a way to reduce classpath (and classloader) to minimal one needed to load script definition and its dependencies + val classpathWithLoader = LazyClasspathWithClassLoader(baseClassLoader) { classpath } + + return scriptTemplatesDiscoverySequence(classpathWithLoader, scriptResolverEnv, messageCollector) +} + +private fun scriptTemplatesDiscoverySequence( + classpathWithLoader: ClasspathWithClassLoader, + scriptResolverEnv: Map, + messageCollector: MessageCollector +): Sequence { + return sequence { + // for jar files the definition class is expected in the same jar as the discovery file + // in case of directories, the class output may come separate from the resources, so some candidates should be deffered and processed later + val defferedDirDependencies = ArrayList() + val defferedDefinitionCandidates = ArrayList() + for (dep in classpathWithLoader.classpath) { + try { + when { + dep.isFile && dep.extension == "jar" -> { // checking for extension is the compiler current behaviour, so the same logic is implemented here + JarFile(dep).use { jar -> + if (jar.getJarEntry(SCRIPT_DEFINITION_MARKERS_PATH) != null) { + val definitionNames = jar.entries().asSequence().mapNotNull { + if (it.isDirectory || !it.name.startsWith(SCRIPT_DEFINITION_MARKERS_PATH)) null + else it.name.removePrefix(SCRIPT_DEFINITION_MARKERS_PATH).removeSuffix( + SCRIPT_DEFINITION_MARKERS_EXTENSION_WITH_DOT + ) + }.toList() + val (loadedDefinitions, notFoundClasses) = + definitionNames.partitionLoadJarDefinitions( + jar, + classpathWithLoader, + scriptResolverEnv, + messageCollector + ) + if (notFoundClasses.isNotEmpty()) { + messageCollector.report( + CompilerMessageSeverity.STRONG_WARNING, + "Configure scripting: unable to find script definitions [${notFoundClasses.joinToString(", ")}]" + ) + } + loadedDefinitions.forEach { + yield(it) + } } - loadedDefinitions.forEach { + } + } + dep.isDirectory -> { + defferedDirDependencies.add(dep) // there is no way to know that the dependency is fully "used" so we add it to the list anyway + val discoveryMarkers = File(dep, SCRIPT_DEFINITION_MARKERS_PATH).listFiles() + if (discoveryMarkers?.isEmpty() == false) { + val (foundDefinitionClasses, notFoundDefinitions) = discoveryMarkers.map { it.name } + .partitionLoadDirDefinitions(dep, classpathWithLoader, scriptResolverEnv, messageCollector) + foundDefinitionClasses.forEach { yield(it) } + defferedDefinitionCandidates.addAll(notFoundDefinitions) } } - } - dep.isDirectory -> { - defferedDirDependencies.add(dep) // there is no way to know that the dependency is fully "used" so we add it to the list anyway - val discoveryDir = File(dep, SCRIPT_DEFINITION_MARKERS_PATH) - if (discoveryDir.isDirectory) { - val (foundDefinitionClasses, notFoundDefinitions) = discoveryDir.listFiles().map { it.name } - .partitionLoadDirDefinitions(dep, loader, scriptResolverEnv, messageCollector) - foundDefinitionClasses.forEach { - yield(it) - } - defferedDefinitionCandidates.addAll(notFoundDefinitions) + else -> { + // assuming that invalid classpath entries will be reported elsewhere anyway, so do not spam user with additional warnings here + messageCollector.report(CompilerMessageSeverity.LOGGING, "Configure scripting: Unknown classpath entry $dep") } } - else -> { - // assuming that invalid classpath entries will be reported elsewhere anyway, so do not spam user with additional warnings here - messageCollector.report(CompilerMessageSeverity.LOGGING, "Configure scripting: Unknown classpath entry $dep") + } catch (e: IOException) { + messageCollector.report( + CompilerMessageSeverity.STRONG_WARNING, "Configure scripting: unable to process classpath entry $dep: $e" + ) + } + } + var remainingDefinitionCandidates: List = defferedDefinitionCandidates + for (dep in defferedDirDependencies) { + if (remainingDefinitionCandidates.isEmpty()) break + try { + val (foundDefinitionClasses, notFoundDefinitions) = + remainingDefinitionCandidates.partitionLoadDirDefinitions(dep, classpathWithLoader, scriptResolverEnv, messageCollector) + foundDefinitionClasses.forEach { + yield(it) } + remainingDefinitionCandidates = notFoundDefinitions + } catch (e: IOException) { + messageCollector.report( + CompilerMessageSeverity.STRONG_WARNING, "Configure scripting: unable to process classpath entry $dep: $e" + ) } - } catch (e: IOException) { + } + if (remainingDefinitionCandidates.isNotEmpty()) { messageCollector.report( - CompilerMessageSeverity.STRONG_WARNING, "Configure scripting: unable to process classpath entry $dep: $e" + CompilerMessageSeverity.STRONG_WARNING, + "The following script definitions are not found in the classpath: [${remainingDefinitionCandidates.joinToString()}]" ) } } - var remainingDefinitionCandidates: List = defferedDefinitionCandidates - for (dep in defferedDirDependencies) { - if (remainingDefinitionCandidates.isEmpty()) break - try { - val (foundDefinitionClasses, notFoundDefinitions) = - remainingDefinitionCandidates.partitionLoadDirDefinitions(dep, loader, scriptResolverEnv, messageCollector) - foundDefinitionClasses.forEach { - yield(it) - } - remainingDefinitionCandidates = notFoundDefinitions - } catch (e: IOException) { - messageCollector.report( - CompilerMessageSeverity.STRONG_WARNING, "Configure scripting: unable to process classpath entry $dep: $e" - ) - } - } - if (remainingDefinitionCandidates.isNotEmpty()) { - messageCollector.report( - CompilerMessageSeverity.STRONG_WARNING, - "The following script definitions are not found in the classpath: [${remainingDefinitionCandidates.joinToString()}]" - ) - } } fun loadScriptTemplatesFromClasspath( @@ -149,7 +183,7 @@ fun loadScriptTemplatesFromClasspath( // then searching the remaining templates in the supplied classpath var remainingTemplates = initialNotFoundTemplates - val classpathAndLoader = + val classpathWithLoader = LazyClasspathWithClassLoader(baseClassLoader) { classpath + dependenciesClasspath } for (dep in classpath) { if (remainingTemplates.isEmpty()) break @@ -158,11 +192,11 @@ fun loadScriptTemplatesFromClasspath( val (loadedDefinitions, notFoundTemplates) = when { dep.isFile && dep.extension == "jar" -> { // checking for extension is the compiler current behaviour, so the same logic is implemented here JarFile(dep).use { jar -> - remainingTemplates.partitionLoadJarDefinitions(jar, classpathAndLoader, scriptResolverEnv, messageCollector) + remainingTemplates.partitionLoadJarDefinitions(jar, classpathWithLoader, scriptResolverEnv, messageCollector) } } dep.isDirectory -> { - remainingTemplates.partitionLoadDirDefinitions(dep, classpathAndLoader, scriptResolverEnv, messageCollector) + remainingTemplates.partitionLoadDirDefinitions(dep, classpathWithLoader, scriptResolverEnv, messageCollector) } else -> { // assuming that invalid classpath entries will be reported elsewhere anyway, so do not spam user with additional warnings here @@ -201,7 +235,7 @@ private data class DefinitionsLoadPartitionResult( ) private inline fun List.partitionLoadDefinitions( - classpathAndLoader: LazyClasspathWithClassLoader, + classpathWithLoader: ClasspathWithClassLoader, scriptResolverEnv: Map, messageCollector: MessageCollector, getBytes: (String) -> ByteArray? @@ -214,14 +248,15 @@ private inline fun List.partitionLoadDefinitions( loadScriptDefinition( it, definitionName, - classpathAndLoader, + classpathWithLoader, scriptResolverEnv, messageCollector ) } when { definition != null -> loaded.add(definition) - classBytes != null -> {} + classBytes != null -> { + } else -> notFound.add(definitionName) } } @@ -230,26 +265,26 @@ private inline fun List.partitionLoadDefinitions( private fun List.partitionLoadJarDefinitions( jar: JarFile, - classpathAndLoader: LazyClasspathWithClassLoader, + classpathWithLoader: ClasspathWithClassLoader, scriptResolverEnv: Map, messageCollector: MessageCollector -): DefinitionsLoadPartitionResult = partitionLoadDefinitions(classpathAndLoader, scriptResolverEnv, messageCollector) { definitionName -> +): DefinitionsLoadPartitionResult = partitionLoadDefinitions(classpathWithLoader, scriptResolverEnv, messageCollector) { definitionName -> jar.getJarEntry("${definitionName.replace('.', '/')}.class")?.let { jar.getInputStream(it).readBytes() } } private fun List.partitionLoadDirDefinitions( dir: File, - classpathAndLoader: LazyClasspathWithClassLoader, + classpathWithLoader: ClasspathWithClassLoader, scriptResolverEnv: Map, messageCollector: MessageCollector -): DefinitionsLoadPartitionResult = partitionLoadDefinitions(classpathAndLoader, scriptResolverEnv, messageCollector) { definitionName -> +): DefinitionsLoadPartitionResult = partitionLoadDefinitions(classpathWithLoader, scriptResolverEnv, messageCollector) { definitionName -> File(dir, "${definitionName.replace('.', '/')}.class").takeIf { it.exists() && it.isFile }?.readBytes() } private fun loadScriptDefinition( templateClassBytes: ByteArray, templateClassName: String, - classpathAndLoader: LazyClasspathWithClassLoader, + classpathWithLoader: ClasspathWithClassLoader, scriptResolverEnv: Map, messageCollector: MessageCollector ): KotlinScriptDefinition? { @@ -260,17 +295,17 @@ private fun loadScriptDefinition( def = LazyScriptDefinitionFromDiscoveredClass( anns, templateClassName, - classpathAndLoader.classpath, + classpathWithLoader.classpath, messageCollector ) } else if (ann.name == ScriptTemplateDefinition::class.simpleName) { - val templateClass = classpathAndLoader.classLoader.loadClass(templateClassName).kotlin - def = KotlinScriptDefinitionFromAnnotatedTemplate(templateClass, scriptResolverEnv, classpathAndLoader.classpath) + val templateClass = classpathWithLoader.classLoader.loadClass(templateClassName).kotlin + def = KotlinScriptDefinitionFromAnnotatedTemplate(templateClass, scriptResolverEnv, classpathWithLoader.classpath) } if (def != null) { messageCollector.report( CompilerMessageSeverity.LOGGING, - "Configure scripting: Added template $templateClassName from ${classpathAndLoader.classpath}" + "Configure scripting: Added template $templateClassName from ${classpathWithLoader.classpath}" ) return def } @@ -322,11 +357,19 @@ private fun loadScriptDefinition( return null } -private class LazyClasspathWithClassLoader(baseClassLoader: ClassLoader, getClasspath: () -> List) { - val classpath by lazy { getClasspath() } - val classLoader by lazy { - URLClassLoader(classpath.map { it.toURI().toURL() }.toTypedArray(), baseClassLoader) - } +private interface ClasspathWithClassLoader { + val classpath: List + val classLoader: ClassLoader +} + +private class SimpleClasspathWithClassLoader( + override val classpath: List, + override val classLoader: ClassLoader +) : ClasspathWithClassLoader + +private class LazyClasspathWithClassLoader(baseClassLoader: ClassLoader?, getClasspath: () -> List) : ClasspathWithClassLoader { + override val classpath by lazy { getClasspath() } + override val classLoader by lazy { URLClassLoader(classpath.map { it.toURI().toURL() }.toTypedArray(), baseClassLoader) } } private inline fun Iterable.partitionMapNotNull(fn: (T) -> R?): Pair, List> {