From 1e74ab5a628ec02509df16085afe82c0c595166d Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Sun, 9 Dec 2018 22:35:39 +0100 Subject: [PATCH] Implement import test, fix import support in compiler and evaluator --- .../kotlin/codegen/context/ScriptContext.kt | 13 ++--- .../cli/jvm/compiler/KotlinCoreEnvironment.kt | 4 +- .../experimental/api/scriptCompilation.kt | 7 ++- .../experimental/host/scriptHostUtil.kt | 4 +- .../jvmhost/impl/KJvmCompiledScript.kt | 52 ++++++++++++------- .../jvmhost/impl/KJvmCompilerImpl.kt | 33 +++++++++--- .../jvmhost/jvmScriptEvaluation.kt | 30 +++++++++-- .../jvmhost/test/ScriptingHostTest.kt | 32 +++++++++++- .../testData/importTest/helloWithVal.kts | 4 ++ .../jvm/impl/BridgeDependenciesResolver.kt | 3 +- 10 files changed, 133 insertions(+), 49 deletions(-) create mode 100644 libraries/scripting/jvm-host/testData/importTest/helloWithVal.kts diff --git a/compiler/backend/src/org/jetbrains/kotlin/codegen/context/ScriptContext.kt b/compiler/backend/src/org/jetbrains/kotlin/codegen/context/ScriptContext.kt index 6fa1d2bad98..ac4b7d0ce4f 100644 --- a/compiler/backend/src/org/jetbrains/kotlin/codegen/context/ScriptContext.kt +++ b/compiler/backend/src/org/jetbrains/kotlin/codegen/context/ScriptContext.kt @@ -16,7 +16,6 @@ package org.jetbrains.kotlin.codegen.context -import org.jetbrains.kotlin.codegen.AsmUtil import org.jetbrains.kotlin.codegen.FieldInfo import org.jetbrains.kotlin.codegen.OwnerKind import org.jetbrains.kotlin.codegen.StackValue @@ -35,7 +34,6 @@ import org.jetbrains.kotlin.resolve.descriptorUtil.getSuperClassNotAny import org.jetbrains.kotlin.resolve.jvm.AsmTypes import org.jetbrains.kotlin.resolve.lazy.descriptors.script.ScriptProvidedPropertiesDescriptor import org.jetbrains.org.objectweb.asm.Type -import kotlin.reflect.KClass class ScriptContext( val typeMapper: KotlinTypeMapper, @@ -83,9 +81,8 @@ class ScriptContext( scriptDescriptor.unsubstitutedPrimaryConstructor.valueParameters[ctorImplicitReceiversParametersStart + index].name.identifier fun getImplicitReceiverType(index: Int): Type? { - val receivers = script.kotlinScriptDefinition.implicitReceivers - val kClass = receivers.getOrNull(index)?.classifier as? KClass<*> - return kClass?.java?.classId?.let(AsmUtil::asmTypeByClassId) + val receiverParam = scriptDescriptor.unsubstitutedPrimaryConstructor.valueParameters[ctorImplicitReceiversParametersStart + index] + return state.typeMapper.mapType(receiverParam.type) } fun getProvidedPropertyName(index: Int): String = @@ -113,10 +110,8 @@ class ScriptContext( fun getScriptFieldName(scriptDescriptor: ScriptDescriptor): String { val index = earlierScripts.indexOf(scriptDescriptor) - if (index < 0) { - throw IllegalStateException("Unregistered script: $scriptDescriptor") - } - return "script$" + (index + 1) + return if (index >= 0) "script$" + (index + 1) + else "\$\$importedScript${scriptDescriptor.name.identifier}" } override fun toString(): String { diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt index de5b0d82fa3..56da504530c 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt @@ -711,7 +711,7 @@ data class ScriptsCompilationDependencies( val sourceDependencies: List ) { data class SourceDependencies( - val script: KtFile, + val scriptFile: KtFile, val sourceDependencies: List ) } @@ -721,7 +721,7 @@ fun collectScriptsCompilationDependencies( configuration: CompilerConfiguration, project: Project, initialSources: Iterable -): ScriptsCompilationDependencies{ +): ScriptsCompilationDependencies { val collectedClassPath = ArrayList() val collectedSources = ArrayList() val collectedSourceDependencies = ArrayList() diff --git a/libraries/scripting/common/src/kotlin/script/experimental/api/scriptCompilation.kt b/libraries/scripting/common/src/kotlin/script/experimental/api/scriptCompilation.kt index 3b4355ab033..4631725b702 100644 --- a/libraries/scripting/common/src/kotlin/script/experimental/api/scriptCompilation.kt +++ b/libraries/scripting/common/src/kotlin/script/experimental/api/scriptCompilation.kt @@ -234,6 +234,9 @@ interface CompiledScript { */ suspend fun getClass(scriptEvaluationConfiguration: ScriptEvaluationConfiguration?): ResultWithDiagnostics> - val importedScripts: List>? - get() = null + /** + * The scripts compiled along with this one in one module, imported or otherwise included into compilation + */ + val otherScripts: List> + get() = emptyList() } diff --git a/libraries/scripting/common/src/kotlin/script/experimental/host/scriptHostUtil.kt b/libraries/scripting/common/src/kotlin/script/experimental/host/scriptHostUtil.kt index 9059b7a01ff..b370594467f 100644 --- a/libraries/scripting/common/src/kotlin/script/experimental/host/scriptHostUtil.kt +++ b/libraries/scripting/common/src/kotlin/script/experimental/host/scriptHostUtil.kt @@ -42,9 +42,9 @@ fun getMergedScriptText(script: SourceCode, configuration: ScriptCompilationConf /** * The implementation of the SourceCode for a script located in a file */ -open class FileScriptSource(val file: File) : ExternalSourceCode { +open class FileScriptSource(val file: File, private val preloadedText: String? = null) : ExternalSourceCode { override val externalLocation: URL get() = file.toURI().toURL() - override val text: String by lazy { file.readText() } + override val text: String by lazy { preloadedText ?: file.readText() } } /** diff --git a/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/impl/KJvmCompiledScript.kt b/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/impl/KJvmCompiledScript.kt index 3f4f10826ad..a233cf8356c 100644 --- a/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/impl/KJvmCompiledScript.kt +++ b/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/impl/KJvmCompiledScript.kt @@ -14,37 +14,49 @@ import kotlin.reflect.KClass import kotlin.script.experimental.api.* import kotlin.script.experimental.jvm.JvmDependency import kotlin.script.experimental.jvmhost.JvmScriptEvaluationConfiguration +import kotlin.script.experimental.jvmhost.actualClassLoader import kotlin.script.experimental.jvmhost.baseClassLoader +class KJvmCompiledModule( + generationState: GenerationState +) : Serializable { + val compilerOutputFiles: Map = + generationState.factory.asList() + .associateTo(sortedMapOf()) { it.relativePath to it.asByteArray() } + + companion object { + @JvmStatic + private val serialVersionUID = 0L + } +} + class KJvmCompiledScript( compilationConfiguration: ScriptCompilationConfiguration, - generationState: GenerationState, private var scriptClassFQName: String, - override val importedScripts: List>? = null + override val otherScripts: List> = emptyList(), + private var compiledModule: KJvmCompiledModule? = null ) : CompiledScript, Serializable { private var _compilationConfiguration: ScriptCompilationConfiguration? = compilationConfiguration - private var compilerOutputFiles: Map = run { - val res = sortedMapOf() - for (it in generationState.factory.asList()) { - res[it.relativePath] = it.asByteArray() - } - res - } override val compilationConfiguration: ScriptCompilationConfiguration get() = _compilationConfiguration!! override suspend fun getClass(scriptEvaluationConfiguration: ScriptEvaluationConfiguration?): ResultWithDiagnostics> = try { - val baseClassLoader = scriptEvaluationConfiguration?.get(JvmScriptEvaluationConfiguration.baseClassLoader) - ?: Thread.currentThread().contextClassLoader - val dependencies = compilationConfiguration[ScriptCompilationConfiguration.dependencies] - ?.flatMap { (it as? JvmDependency)?.classpath?.map { it.toURI().toURL() } ?: emptyList() } - // TODO: previous dependencies and classloaders should be taken into account here - val classLoaderWithDeps = - if (dependencies == null) baseClassLoader - else URLClassLoader(dependencies.toTypedArray(), baseClassLoader) - val classLoader = CompiledScriptClassLoader(classLoaderWithDeps, compilerOutputFiles) + val classLoader = scriptEvaluationConfiguration?.get(JvmScriptEvaluationConfiguration.actualClassLoader) + ?: run { + if (compiledModule == null) + return ResultWithDiagnostics.Failure("Unable to load class $scriptClassFQName: no compiled module is provided".asErrorDiagnostics()) + val baseClassLoader = scriptEvaluationConfiguration?.get(JvmScriptEvaluationConfiguration.baseClassLoader) + ?: Thread.currentThread().contextClassLoader + val dependencies = compilationConfiguration[ScriptCompilationConfiguration.dependencies] + ?.flatMap { (it as? JvmDependency)?.classpath?.map { it.toURI().toURL() } ?: emptyList() } + // TODO: previous dependencies and classloaders should be taken into account here + val classLoaderWithDeps = + if (dependencies == null) baseClassLoader + else URLClassLoader(dependencies.toTypedArray(), baseClassLoader) + CompiledScriptClassLoader(classLoaderWithDeps, compiledModule!!.compilerOutputFiles) + } val clazz = classLoader.loadClass(scriptClassFQName).kotlin clazz.asSuccess() @@ -65,13 +77,13 @@ class KJvmCompiledScript( } private fun writeObject(outputStream: ObjectOutputStream) { - outputStream.writeObject(compilerOutputFiles) + outputStream.writeObject(compiledModule) outputStream.writeObject(scriptClassFQName) } private fun readObject(inputStream: ObjectInputStream) { _compilationConfiguration = null - compilerOutputFiles = inputStream.readObject() as Map + compiledModule = inputStream.readObject() as KJvmCompiledModule scriptClassFQName = inputStream.readObject() as String } diff --git a/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/impl/KJvmCompilerImpl.kt b/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/impl/KJvmCompilerImpl.kt index 6821229f2cd..42ae6cc7821 100644 --- a/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/impl/KJvmCompilerImpl.kt +++ b/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/impl/KJvmCompilerImpl.kt @@ -38,6 +38,7 @@ import org.jetbrains.kotlin.script.KotlinScriptDefinition import org.jetbrains.kotlin.script.util.KotlinJars import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstanceOrNull import java.io.File +import java.util.* import kotlin.reflect.KClass import kotlin.reflect.KType import kotlin.reflect.full.starProjectedType @@ -172,20 +173,38 @@ class KJvmCompilerImpl(val hostConfiguration: ScriptingHostConfiguration) : KJvm sourceFiles, kotlinCompilerConfiguration ).build() + KotlinCodegenFacade.compileCorrectFiles(generationState, CompilationErrorHandler.THROW_EXCEPTION) - fun makeCompiledScript(script: KtScript): KJvmCompiledScript<*> { + val scriptDependenciesStack = ArrayDeque() - val importedScripts = sourceDependencies.find { it.script == script }?.sourceDependencies?.mapNotNull { sourceFile -> - sourceFile.declarations.firstIsInstanceOrNull()?.let { makeCompiledScript(it) } - } ?: emptyList() + fun makeOtherScripts(script: KtScript): List> { - return KJvmCompiledScript(updatedConfiguration, generationState, script.fqName.asString(), importedScripts) + // TODO: ensure that it is caught earlier (as well) since it would be more economical + if (scriptDependenciesStack.contains(script)) + throw IllegalArgumentException("Unable to handle recursive script dependencies") + scriptDependenciesStack.push(script) + + val containingKtFile = script.containingKtFile + val otherScripts: List> = + sourceDependencies.find { it.scriptFile == containingKtFile }?.sourceDependencies?.mapNotNull { sourceFile -> + sourceFile.declarations.firstIsInstanceOrNull()?.let { + KJvmCompiledScript(updatedConfiguration, it.fqName.asString(), makeOtherScripts(it)) + } + } ?: emptyList() + + scriptDependenciesStack.pop() + return otherScripts } - val res = makeCompiledScript(ktScript) + val compiledScript = KJvmCompiledScript( + updatedConfiguration, + ktScript.fqName.asString(), + makeOtherScripts(ktScript), + KJvmCompiledModule(generationState) + ) - return ResultWithDiagnostics.Success(res, messageCollector.diagnostics) + return ResultWithDiagnostics.Success(compiledScript, messageCollector.diagnostics) } catch (ex: Throwable) { return failure(ex.asDiagnostics()) } diff --git a/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/jvmScriptEvaluation.kt b/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/jvmScriptEvaluation.kt index 43b09afcb48..c7d1b4fa1a8 100644 --- a/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/jvmScriptEvaluation.kt +++ b/libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/jvmScriptEvaluation.kt @@ -17,6 +17,8 @@ open class JvmScriptEvaluationConfiguration : PropertiesCollection.Builder() { val JvmScriptEvaluationConfiguration.baseClassLoader by PropertiesCollection.key(Thread.currentThread().contextClassLoader) +val JvmScriptEvaluationConfiguration.actualClassLoader by PropertiesCollection.key() + val ScriptEvaluationConfiguration.jvm get() = JvmScriptEvaluationConfiguration() open class BasicJvmScriptEvaluator : ScriptEvaluator { @@ -43,12 +45,30 @@ open class BasicJvmScriptEvaluator : ScriptEvaluator { } val importedScriptsReports = ArrayList() var importedScriptsLoadingFailed = false - compiledScript.importedScripts?.forEach { - val importedScriptEvalRes = invoke(it, scriptEvaluationConfiguration) + + // for other scripts we need evaluation configuration with actualClassloader set, + // so they are loaded in the same classloader as the "main" script + val updatedEvalConfiguration = when { + scriptEvaluationConfiguration == null -> ScriptEvaluationConfiguration { + // TODO: find out why dsl syntax doesn't work here + set(JvmScriptEvaluationConfiguration.actualClassLoader, scriptClass.java.classLoader) + } + scriptEvaluationConfiguration.getNoDefault(JvmScriptEvaluationConfiguration.actualClassLoader) == null -> ScriptEvaluationConfiguration(scriptEvaluationConfiguration) { + // TODO: find out why dsl syntax doesn't work here + set(JvmScriptEvaluationConfiguration.actualClassLoader, scriptClass.java.classLoader) + } + else -> scriptEvaluationConfiguration + } + + compiledScript.otherScripts.forEach { + // TODO: in the future other scripts could be used for other purposes, so args here should be added only for actually imported scripts + // (it means that we should keep mapping somewhere (or reuse one with source dependencies) between imported scrips and e.g. fqnames) + val importedScriptEvalRes = invoke(it, updatedEvalConfiguration) importedScriptsReports.addAll(importedScriptEvalRes.reports) when (importedScriptEvalRes) { is ResultWithDiagnostics.Success -> { - args.add(importedScriptEvalRes.value) + // TODO: checks and diagnostics + args.add((importedScriptEvalRes.value.returnValue as ResultValue.Value).value) } else -> { importedScriptsLoadingFailed = true @@ -60,14 +80,14 @@ open class BasicJvmScriptEvaluator : ScriptEvaluator { ResultWithDiagnostics.Failure(importedScriptsReports) } else { - scriptEvaluationConfiguration?.get(ScriptEvaluationConfiguration.constructorArgs)?.let { + updatedEvalConfiguration[ScriptEvaluationConfiguration.constructorArgs]?.let { args.addAll(it) } val ctor = scriptClass.java.constructors.single() val instance = ctor.newInstance(*args.toArray()) // TODO: fix result value - ResultWithDiagnostics.Success(EvaluationResult(ResultValue.Value("", instance, ""), scriptEvaluationConfiguration)) + ResultWithDiagnostics.Success(EvaluationResult(ResultValue.Value("", instance, ""), updatedEvalConfiguration)) } } } diff --git a/libraries/scripting/jvm-host/test/kotlin/script/experimental/jvmhost/test/ScriptingHostTest.kt b/libraries/scripting/jvm-host/test/kotlin/script/experimental/jvmhost/test/ScriptingHostTest.kt index 1d19eb67a3d..61483d12d81 100644 --- a/libraries/scripting/jvm-host/test/kotlin/script/experimental/jvmhost/test/ScriptingHostTest.kt +++ b/libraries/scripting/jvm-host/test/kotlin/script/experimental/jvmhost/test/ScriptingHostTest.kt @@ -15,6 +15,7 @@ import java.security.MessageDigest import kotlin.reflect.KClass import kotlin.script.experimental.api.* import kotlin.script.experimental.host.BasicScriptingHost +import kotlin.script.experimental.host.FileScriptSource import kotlin.script.experimental.host.toScriptSource import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration import kotlin.script.experimental.jvmhost.* @@ -50,6 +51,31 @@ class ScriptingHostTest : TestCase() { Assert.assertEquals(greeting, output) } + @Test + fun testSimpleImport() { + val greeting = "Hello from helloWithVal script!\nHello from imported helloWithVal script!" + val script = "println(\"Hello from imported \$helloScriptName script!\")" + val compilationConfiguration = createJvmCompilationConfigurationFromTemplate { + refineConfiguration { + beforeCompiling { ctx -> + val importedScript = File(TEST_DATA_DIR, "importTest/helloWithVal.kts") + if ((ctx.script as? FileScriptSource)?.file?.canonicalFile == importedScript.canonicalFile) { + ctx.compilationConfiguration + } else { + ScriptCompilationConfiguration(ctx.compilationConfiguration) { + importScripts(importedScript.toScriptSource()) + } + }.asSuccess() + } + } + } + val output = captureOut { + BasicJvmScriptingHost().eval(script.toScriptSource(), compilationConfiguration, null).throwOnFailure() + } + Assert.assertEquals(greeting, output) + } + + @Test fun testMemoryCache() { val script = "val x = 1\nprintln(\"x = \$x\")" @@ -159,7 +185,11 @@ class ScriptingHostTest : TestCase() { fun ResultWithDiagnostics<*>.throwOnFailure(): ResultWithDiagnostics<*> = apply { if (this is ResultWithDiagnostics.Failure) { - throw Exception("Compilation/evaluation failed:\n ${reports.joinToString("\n ") { it.exception?.toString() ?: it.message }}") + val firstExceptionFromReports = reports.find { it.exception != null }?.exception + throw Exception( + "Compilation/evaluation failed:\n ${reports.joinToString("\n ") { it.exception?.toString() ?: it.message }}", + firstExceptionFromReports + ) } } diff --git a/libraries/scripting/jvm-host/testData/importTest/helloWithVal.kts b/libraries/scripting/jvm-host/testData/importTest/helloWithVal.kts new file mode 100644 index 00000000000..6fbf8bc605c --- /dev/null +++ b/libraries/scripting/jvm-host/testData/importTest/helloWithVal.kts @@ -0,0 +1,4 @@ + +val helloScriptName = "helloWithVal" + +println("Hello from $helloScriptName script!") diff --git a/libraries/scripting/jvm/src/kotlin/script/experimental/jvm/impl/BridgeDependenciesResolver.kt b/libraries/scripting/jvm/src/kotlin/script/experimental/jvm/impl/BridgeDependenciesResolver.kt index b07189950d1..527aedf6ba8 100644 --- a/libraries/scripting/jvm/src/kotlin/script/experimental/jvm/impl/BridgeDependenciesResolver.kt +++ b/libraries/scripting/jvm/src/kotlin/script/experimental/jvm/impl/BridgeDependenciesResolver.kt @@ -14,6 +14,7 @@ import kotlin.script.experimental.dependencies.AsyncDependenciesResolver import kotlin.script.experimental.dependencies.DependenciesResolver import kotlin.script.experimental.dependencies.ScriptDependencies import kotlin.script.experimental.dependencies.ScriptReport +import kotlin.script.experimental.host.FileScriptSource import kotlin.script.experimental.host.toScriptSource import kotlin.script.experimental.jvm.JvmDependency import kotlin.script.experimental.jvm.compat.mapToLegacyScriptReportPosition @@ -94,8 +95,8 @@ internal fun List.mapScriptReportsToDiagnostics() = map { ScriptReport(it.message, mapToLegacyScriptReportSeverity(it.severity), mapToLegacyScriptReportPosition(it.location)) } internal fun ScriptContents.toScriptSource(): SourceCode = when { + file != null -> FileScriptSource(file!!, text?.toString()) text != null -> text!!.toString().toScriptSource() - file != null -> file!!.toScriptSource() else -> throw IllegalArgumentException("Unable to convert script contents $this into script source") }