Avoid using reflected types in the scripting API

since it causes numerous classloading issues. Using the wrapping types
and reload them in the proper context when needed.
Note: this version supports only classes, but the wrapping type could
be extended to support other types in the future.
+ numerous fixes related to proper loading and handling of the templates.
This commit is contained in:
Ilya Chernikov
2018-05-22 19:22:42 +02:00
parent 4d65f0478b
commit a46dd5b30e
28 changed files with 506 additions and 185 deletions
@@ -10,13 +10,11 @@ import org.jetbrains.kotlin.idea.KotlinFileType
import org.jetbrains.kotlin.name.Name
import org.jetbrains.kotlin.name.NameUtils
import org.jetbrains.kotlin.psi.KtScript
import org.jetbrains.kotlin.script.*
import org.jetbrains.kotlin.script.KotlinScriptDefinition
import kotlin.reflect.KClass
import kotlin.reflect.KType
import kotlin.script.experimental.api.ScriptCompileConfigurationProperties
import kotlin.script.experimental.api.ScriptDefinition
import kotlin.script.experimental.api.ScriptDefinitionProperties
import kotlin.script.experimental.api.ScriptingEnvironmentProperties
import kotlin.reflect.full.starProjectedType
import kotlin.script.experimental.api.*
import kotlin.script.experimental.dependencies.DependenciesResolver
import kotlin.script.experimental.jvm.impl.BridgeDependenciesResolver
import kotlin.script.experimental.location.ScriptExpectedLocation
@@ -28,8 +26,9 @@ abstract class KotlinScriptDefinitionAdapterFromNewAPIBase : KotlinScriptDefinit
protected abstract val scriptFileExtensionWithDot: String
open val baseClass: KClass<*>
get() = scriptDefinition.compilationConfigurator.defaultConfiguration[ScriptingEnvironmentProperties.baseClass]
open val baseClass: KClass<*> by lazy {
getScriptingClass(scriptDefinition.compilationConfigurator.defaultConfiguration[ScriptingEnvironmentProperties.baseClass])
}
override val template: KClass<*> get() = baseClass
@@ -54,17 +53,22 @@ abstract class KotlinScriptDefinitionAdapterFromNewAPIBase : KotlinScriptDefinit
}
override val acceptedAnnotations: List<KClass<out Annotation>> by lazy {
scriptDefinition.compilationConfigurator.defaultConfiguration.getOrNull(ScriptCompileConfigurationProperties.refineConfigurationOnAnnotations)
?: emptyList()
val annNames =
scriptDefinition.compilationConfigurator.defaultConfiguration.getOrNull(ScriptCompileConfigurationProperties.refineConfigurationOnAnnotations)
?: emptyList()
annNames.map { getScriptingClass(it) as KClass<out Annotation> }
}
override val implicitReceivers: List<KType> by lazy {
scriptDefinition.compilationConfigurator.defaultConfiguration.getOrNull(ScriptCompileConfigurationProperties.scriptImplicitReceivers)
?: emptyList()
val rcNames =
scriptDefinition.compilationConfigurator.defaultConfiguration.getOrNull(ScriptCompileConfigurationProperties.scriptImplicitReceivers)
?: emptyList()
rcNames.map { getScriptingClass(it).starProjectedType }
}
override val environmentVariables: List<Pair<String, KType>> by lazy {
scriptDefinition.compilationConfigurator.defaultConfiguration.getOrNull(ScriptCompileConfigurationProperties.contextVariables)?.map { (k, v) -> k to v }
scriptDefinition.compilationConfigurator.defaultConfiguration.getOrNull(ScriptCompileConfigurationProperties.contextVariables)
?.map { (k, v) -> k to getScriptingClass(v).starProjectedType }
?: emptyList()
}
@@ -77,6 +81,18 @@ abstract class KotlinScriptDefinitionAdapterFromNewAPIBase : KotlinScriptDefinit
ScriptExpectedLocation.SourcesOnly,
ScriptExpectedLocation.TestsOnly
)
private val scriptingClassGetter by lazy {
scriptDefinition.properties.getOrNull(ScriptingEnvironmentProperties.getScriptingClass)
?: throw IllegalArgumentException("Expecting 'getScriptingClass' property in the scripting environment")
}
private fun getScriptingClass(type: KotlinType) =
scriptingClassGetter(
type,
KotlinScriptDefinition::class, // Assuming that the KotlinScriptDefinition class is loaded in the proper classloader
scriptDefinition.properties
)
}
@@ -12,33 +12,27 @@ package org.jetbrains.kotlin.scripting.compiler.plugin
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.org.objectweb.asm.AnnotationVisitor
import org.jetbrains.org.objectweb.asm.ClassReader
import org.jetbrains.org.objectweb.asm.ClassVisitor
import org.jetbrains.org.objectweb.asm.Opcodes
import java.io.File
import java.net.URLClassLoader
import kotlin.script.experimental.annotations.KotlinScript
import kotlin.script.experimental.annotations.KotlinScriptFileExtension
import kotlin.script.experimental.api.*
import kotlin.script.experimental.definitions.ScriptDefinitionFromAnnotatedBaseClass
import kotlin.script.experimental.jvm.JvmDependency
import kotlin.script.experimental.jvm.JvmGetScriptingClass
class LazyScriptDefinitionFromDiscoveredClass(
classBytes: ByteArray,
class LazyScriptDefinitionFromDiscoveredClass internal constructor(
private val annotationsFromAsm: ArrayList<BinAnnData>,
private val className: String,
private val classpath: List<File>,
private val messageCollector: MessageCollector
) : KotlinScriptDefinitionAdapterFromNewAPIBase() {
private val annotationsFromAsm = loadAnnotationsFromClass(classBytes)
private val classloader by lazy {
// should use this cl to allow smooth interop with classes explicitly mentioned here, see e.g. scriptDefinition body
val parentClassloader = LazyScriptDefinitionFromDiscoveredClass::class.java.classLoader
if (classpath.isEmpty()) parentClassloader
else URLClassLoader(classpath.map { it.toURI().toURL() }.toTypedArray(), parentClassloader)
}
constructor(
classBytes: ByteArray,
className: String,
classpath: List<File>,
messageCollector: MessageCollector
) : this(loadAnnotationsFromClass(classBytes), className, classpath, messageCollector)
override val scriptDefinition: ScriptDefinition by lazy {
messageCollector.report(
@@ -46,10 +40,11 @@ class LazyScriptDefinitionFromDiscoveredClass(
"Configure scripting: loading script definition class $className using classpath $classpath\n. ${Thread.currentThread().stackTrace}"
)
try {
val cls = classloader.loadClass(className).kotlin
ScriptDefinitionFromAnnotatedBaseClass(
ScriptingEnvironment(
ScriptingEnvironmentProperties.baseClass to cls
ScriptingEnvironmentProperties.baseClass to KotlinType(className),
ScriptingEnvironmentProperties.configurationDependencies to listOf(JvmDependency(classpath)),
ScriptingEnvironmentProperties.getScriptingClass to JvmGetScriptingClass()
)
)
} catch (ex: ClassNotFoundException) {
@@ -86,41 +81,3 @@ object InvalidScriptDefinition : ScriptDefinition {
override val evaluator: ScriptEvaluator<*>? = null
}
private class BinAnnData(
val name: String,
val args: ArrayList<String> = arrayListOf()
)
private class TemplateAnnotationVisitor(val anns: ArrayList<BinAnnData> = arrayListOf()) : AnnotationVisitor(Opcodes.ASM5) {
override fun visit(name: String?, value: Any?) {
anns.last().args.add(value.toString())
}
}
private class TemplateClassVisitor(val annVisitor: TemplateAnnotationVisitor) : ClassVisitor(Opcodes.ASM5) {
override fun visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor {
val shortName = jvmDescToClassId(desc).shortClassName.asString()
if (shortName.startsWith("KotlinScript")) {
annVisitor.anns.add(BinAnnData(shortName))
}
return annVisitor
}
}
private fun jvmDescToClassId(desc: String): ClassId {
assert(desc.startsWith("L") && desc.endsWith(";")) { "Not a JVM descriptor: $desc" }
val name = desc.substring(1, desc.length - 1)
val cid = ClassId.topLevel(FqName(name.replace('/', '.')))
return cid
}
private fun loadAnnotationsFromClass(fileContents: ByteArray): ArrayList<BinAnnData> {
val visitor =
TemplateClassVisitor(TemplateAnnotationVisitor())
ClassReader(fileContents).accept(visitor, ClassReader.SKIP_CODE or ClassReader.SKIP_DEBUG or ClassReader.SKIP_FRAMES)
return visitor.annVisitor.anns
}
@@ -17,15 +17,19 @@ import java.net.URLClassLoader
import java.util.jar.JarFile
import kotlin.coroutines.experimental.buildSequence
import kotlin.script.experimental.annotations.KotlinScript
import kotlin.script.experimental.api.KotlinType
import kotlin.script.experimental.api.ScriptingEnvironment
import kotlin.script.experimental.api.ScriptingEnvironmentProperties
import kotlin.script.experimental.definitions.ScriptDefinitionFromAnnotatedBaseClass
import kotlin.script.experimental.jvm.JvmGetScriptingClass
import kotlin.script.templates.ScriptTemplateDefinition
internal const val SCRIPT_DEFINITION_MARKERS_PATH = "META-INF/kotlin/script/templates/"
class ScriptDefinitionsFromClasspathDiscoverySource(
private val classpath: List<File>,
private val defaultScriptDefinitionClasspath: List<File>,
private val scriptResolverEnv: Map<String, Any?>,
private val messageCollector: MessageCollector
) : ScriptDefinitionsSource {
@@ -33,21 +37,28 @@ class ScriptDefinitionsFromClasspathDiscoverySource(
discoverScriptTemplatesInClasspath(
classpath,
defaultScriptDefinitionClasspath,
this::class.java.classLoader,
scriptResolverEnv,
messageCollector
)
}
}
internal fun discoverScriptTemplatesInClasspath(
classpath: Iterable<File>,
classpath: List<File>,
defaultScriptDefinitionClasspath: List<File>,
baseClassLoader: ClassLoader,
scriptResolverEnv: Map<String, Any?>,
messageCollector: MessageCollector
): Sequence<LazyScriptDefinitionFromDiscoveredClass> = buildSequence {
): Sequence<KotlinScriptDefinition> = buildSequence {
// TODO: try to find a way to reduce classpath (and classloader) to minimal one needed to load script definition and its dependencies
val classLoader by lazy {
URLClassLoader(classpath.map { it.toURI().toURL() }.toTypedArray(), baseClassLoader)
}
for (dep in classpath) {
try {
when {
// checking for extension is the compiler current behaviour, so the same logic is implemented here
dep.isFile && dep.extension == "jar" -> {
dep.isFile && dep.extension == "jar" -> { // checking for extension is the compiler current behaviour, so the same logic is implemented here
val jar = JarFile(dep)
if (jar.getJarEntry(SCRIPT_DEFINITION_MARKERS_PATH) != null) {
for (template in jar.entries()) {
@@ -60,17 +71,16 @@ internal fun discoverScriptTemplatesInClasspath(
"Configure scripting: class not found $templateClassName"
)
} else {
messageCollector.report(
CompilerMessageSeverity.LOGGING,
"Configure scripting: Added template $templateClassName from $dep"
)
yield(
LazyScriptDefinitionFromDiscoveredClass(
jar.getInputStream(templateClass).readBytes(),
templateClassName, listOf(dep) + jar.extractClasspath(defaultScriptDefinitionClasspath),
messageCollector
loadScriptDefinition(
jar.getInputStream(templateClass).readBytes(),
templateClassName, classpath, { classLoader }, scriptResolverEnv, messageCollector
)?.let {
messageCollector.report(
CompilerMessageSeverity.LOGGING,
"Configure scripting: Added template $templateClassName from $dep"
)
)
yield(it)
}
}
}
}
@@ -79,25 +89,30 @@ internal fun discoverScriptTemplatesInClasspath(
dep.isDirectory -> {
val dir = File(dep, SCRIPT_DEFINITION_MARKERS_PATH)
if (dir.isDirectory) {
dir.listFiles().forEach {
val templateClass = File(dep, "${it.name.replace('.', '/')}.class")
if (!templateClass.exists() || !templateClass.isFile) {
val templateClasspath by lazy {
listOf(dep) + defaultScriptDefinitionClasspath
}
val classLoader by lazy {
URLClassLoader(templateClasspath.map { it.toURI().toURL() }.toTypedArray(), baseClassLoader)
}
dir.listFiles().forEach { templateClassNmae ->
val templateClassFile = File(dep, "${templateClassNmae.name.replace('.', '/')}.class")
if (!templateClassFile.exists() || !templateClassFile.isFile) {
messageCollector.report(
CompilerMessageSeverity.WARNING,
"Configure scripting: class not found ${it.name}"
"Configure scripting: class not found ${templateClassNmae.name}"
)
} else {
messageCollector.report(
CompilerMessageSeverity.LOGGING,
"Configure scripting: Added template ${it.name} from $dep"
)
yield(
LazyScriptDefinitionFromDiscoveredClass(
templateClass.readBytes(),
it.name, listOf(dep) + defaultScriptDefinitionClasspath,
messageCollector
loadScriptDefinition(
templateClassFile.readBytes(),
templateClassNmae.name, templateClasspath, { classLoader }, scriptResolverEnv, messageCollector
)?.let {
messageCollector.report(
CompilerMessageSeverity.LOGGING,
"Configure scripting: Added template ${templateClassNmae.name} from $dep"
)
)
yield(it)
}
}
}
}
@@ -119,23 +134,138 @@ internal fun discoverScriptTemplatesInClasspath(
}
}
internal fun loadScriptTemplatesFromClasspath(
scriptTemplates: List<String>,
classpath: List<File>,
dependenciesClasspath: List<File>,
baseClassLoader: ClassLoader,
scriptResolverEnv: Map<String, Any?>,
messageCollector: MessageCollector
): Sequence<KotlinScriptDefinition> = buildSequence {
val templatesLeftToFind = ArrayList<String>()
// trying the direct classloading from baseClassloader first, since this is the most performant variant
for (template in scriptTemplates) {
val def = loadScriptDefinition(baseClassLoader, template, scriptResolverEnv, messageCollector)
if (def == null) {
templatesLeftToFind.add(template)
} else {
yield(def!!)
}
}
// then searching the remaining templates in the supplied classpath
if (templatesLeftToFind.isNotEmpty()) {
val templateClasspath by lazy {
classpath + dependenciesClasspath
}
val classLoader by lazy {
URLClassLoader(templateClasspath.map { it.toURI().toURL() }.toTypedArray(), baseClassLoader)
}
for (dep in classpath) {
try {
when {
dep.isFile && dep.extension == "jar" -> { // checking for extension is the compiler current behaviour, so the same logic is implemented here
val jar = JarFile(dep)
for (templateClassName in templatesLeftToFind) {
val templateClassEntry = jar.getJarEntry("${templateClassName.replace('.', '/')}.class")
if (templateClassEntry != null) {
loadScriptDefinition(
jar.getInputStream(templateClassEntry).readBytes(),
templateClassName, templateClasspath, { classLoader }, scriptResolverEnv, messageCollector
)?.let {
templatesLeftToFind.remove(templateClassName)
yield(it)
}
}
}
}
dep.isDirectory -> {
for (templateClassName in scriptTemplates) {
val templateClassFile = File(dep, "${templateClassName.replace('.', '/')}.class")
if (templateClassFile.exists()) {
loadScriptDefinition(
templateClassFile.readBytes(),
templateClassName, templateClasspath, { classLoader }, scriptResolverEnv, messageCollector
)?.let {
templatesLeftToFind.remove(templateClassName)
yield(it)
}
}
}
}
else -> {
// assuming that invalid classpath entries will be reported elsewhere anyway, so do not spam user with additional warnings here
messageCollector.report(
CompilerMessageSeverity.LOGGING,
"Configure scripting: Unknown classpath entry $dep"
)
}
}
} catch (e: IOException) {
messageCollector.report(
CompilerMessageSeverity.WARNING,
"Configure scripting: unable to process classpath entry $dep: $e"
)
}
}
}
if (templatesLeftToFind.isNotEmpty()) {
messageCollector.report(
CompilerMessageSeverity.WARNING,
"Configure scripting: unable to find script definition classes: $templatesLeftToFind"
)
}
}
private fun loadScriptDefinition(
templateClassBytes: ByteArray,
templateClassName: String,
templateClasspath: List<File>,
getClassLoader: () -> ClassLoader,
scriptResolverEnv: Map<String, Any?>,
messageCollector: MessageCollector
): KotlinScriptDefinition? {
val anns = loadAnnotationsFromClass(templateClassBytes)
for (ann in anns) {
var def: KotlinScriptDefinition? = null
if (ann.name == KotlinScript::class.simpleName) {
def = LazyScriptDefinitionFromDiscoveredClass(anns, templateClassName, templateClasspath, messageCollector)
} else if (ann.name == ScriptTemplateDefinition::class.simpleName) {
val templateClass = getClassLoader().loadClass(templateClassName).kotlin
def = KotlinScriptDefinitionFromAnnotatedTemplate(templateClass, scriptResolverEnv, templateClasspath)
}
if (def != null) {
messageCollector.report(
CompilerMessageSeverity.LOGGING,
"Configure scripting: Added template $templateClassName from $templateClasspath"
)
return def
}
}
messageCollector.report(
CompilerMessageSeverity.WARNING,
"Configure scripting: $templateClassName is not marked with any known kotlin script annotation"
)
return null
}
private fun JarFile.extractClasspath(defaultClasspath: List<File>): List<File> =
manifest.mainAttributes.getValue("Class-Path")?.split(" ")?.map(::File) ?: defaultClasspath
internal fun loadScriptDefinition(
classloader: URLClassLoader,
private fun loadScriptDefinition(
classLoader: ClassLoader,
template: String,
scriptResolverEnv: Map<String, Any?>,
messageCollector: MessageCollector
): KotlinScriptDefinition? {
try {
val cls = classloader.loadClass(template)
val cls = classLoader.loadClass(template)
val def =
if (cls.annotations.firstIsInstanceOrNull<KotlinScript>() != null) {
KotlinScriptDefinitionAdapterFromNewAPI(
ScriptDefinitionFromAnnotatedBaseClass(
ScriptingEnvironment(
ScriptingEnvironmentProperties.baseClass to cls.kotlin
ScriptingEnvironmentProperties.baseClass to KotlinType(cls.kotlin),
ScriptingEnvironmentProperties.getScriptingClass to JvmGetScriptingClass()
)
)
)
@@ -149,7 +279,7 @@ internal fun loadScriptDefinition(
)
return def
} catch (ex: ClassNotFoundException) {
messageCollector.report(CompilerMessageSeverity.ERROR, "Cannot find script definition template class $template")
// return null
} catch (ex: Exception) {
messageCollector.report(
CompilerMessageSeverity.ERROR,
@@ -7,7 +7,6 @@ package org.jetbrains.kotlin.scripting.compiler.plugin
import com.intellij.mock.MockProject
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
import org.jetbrains.kotlin.cli.common.messages.MessageCollector
import org.jetbrains.kotlin.cli.jvm.config.jvmClasspathRoots
import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar
@@ -16,7 +15,6 @@ import org.jetbrains.kotlin.config.JVMConfigurationKeys
import org.jetbrains.kotlin.extensions.CompilerConfigurationExtension
import org.jetbrains.kotlin.script.StandardScriptDefinition
import java.io.File
import java.net.URLClassLoader
class ScriptingCompilerConfigurationExtension(val project: MockProject) : CompilerConfigurationExtension {
@@ -41,6 +39,7 @@ class ScriptingCompilerConfigurationExtension(val project: MockProject) : Compil
configureScriptDefinitions(
explicitScriptDefinitions,
configuration,
this::class.java.classLoader,
messageCollector,
scriptResolverEnv
)
@@ -57,6 +56,7 @@ class ScriptingCompilerConfigurationExtension(val project: MockProject) : Compil
ScriptDefinitionsFromClasspathDiscoverySource(
configuration.jvmClasspathRoots,
emptyList(),
configuration.get(ScriptingConfigurationKeys.LEGACY_SCRIPT_RESOLVER_ENVIRONMENT_OPTION) ?: emptyMap(),
messageCollector
)
)
@@ -74,31 +74,17 @@ class ScriptingCompilerConfigurationComponentRegistrar : ComponentRegistrar {
fun configureScriptDefinitions(
scriptTemplates: List<String>,
configuration: CompilerConfiguration,
baseClassloader: ClassLoader,
messageCollector: MessageCollector,
scriptResolverEnv: Map<String, Any?>
) {
val classpath = configuration.jvmClasspathRoots
// TODO: consider using escaping to allow kotlin escaped names in class names
if (scriptTemplates.isNotEmpty()) {
val classloader =
URLClassLoader(classpath.map { it.toURI().toURL() }.toTypedArray(), Thread.currentThread().contextClassLoader)
var hasErrors = false
for (template in scriptTemplates) {
val def = loadScriptDefinition(
classloader,
template,
scriptResolverEnv,
messageCollector
)
if (!hasErrors && def == null) hasErrors = true
if (def != null) {
configuration.add(JVMConfigurationKeys.SCRIPT_DEFINITIONS, def)
loadScriptTemplatesFromClasspath(scriptTemplates, classpath, emptyList(), baseClassloader, scriptResolverEnv, messageCollector)
.forEach {
configuration.add(JVMConfigurationKeys.SCRIPT_DEFINITIONS, it)
}
}
if (hasErrors) {
messageCollector.report(CompilerMessageSeverity.LOGGING, "(Classpath used for templates loading: $classpath)")
return
}
}
}
@@ -0,0 +1,52 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.scripting.compiler.plugin
import org.jetbrains.kotlin.name.ClassId
import org.jetbrains.kotlin.name.FqName
import org.jetbrains.org.objectweb.asm.AnnotationVisitor
import org.jetbrains.org.objectweb.asm.ClassReader
import org.jetbrains.org.objectweb.asm.ClassVisitor
import org.jetbrains.org.objectweb.asm.Opcodes
internal class BinAnnData(
val name: String,
val args: ArrayList<String> = arrayListOf()
)
private class TemplateAnnotationVisitor(val anns: ArrayList<BinAnnData> = arrayListOf()) : AnnotationVisitor(Opcodes.ASM5) {
override fun visit(name: String?, value: Any?) {
anns.last().args.add(value.toString())
}
}
private class TemplateClassVisitor(val annVisitor: TemplateAnnotationVisitor) : ClassVisitor(Opcodes.ASM5) {
override fun visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor {
val shortName = jvmDescToClassId(desc).shortClassName.asString()
if (shortName.startsWith("KotlinScript")) {
annVisitor.anns.add(BinAnnData(shortName))
}
return annVisitor
}
}
private fun jvmDescToClassId(desc: String): ClassId {
assert(desc.startsWith("L") && desc.endsWith(";")) { "Not a JVM descriptor: $desc" }
val name = desc.substring(1, desc.length - 1)
val cid = ClassId.topLevel(FqName(name.replace('/', '.')))
return cid
}
internal fun loadAnnotationsFromClass(fileContents: ByteArray): ArrayList<BinAnnData> {
val visitor =
TemplateClassVisitor(TemplateAnnotationVisitor())
ClassReader(fileContents).accept(visitor, ClassReader.SKIP_CODE or ClassReader.SKIP_DEBUG or ClassReader.SKIP_FRAMES)
return visitor.annVisitor.anns
}