azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scripting invalidate compiled cache if a dependency is missing

Browse Source 
#KT-48303 fixed
This commit is contained in:
Ilya Chernikov
2021-08-18 12:58:44 +02:00
committed by TeamCityServer
parent e60e80f19a
commit 61e5f68b8d
4 changed files with 146 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/scripting/jvm-host-test/test/kotlin/script/experimental/jvmhost/test/CachingTest.kt
+76 -13
View File
@@ -7,13 +7,13 @@ package kotlin.script.experimental.jvmhost.test
import junit.framework.TestCase
import kotlinx.coroutines.runBlocking
import org.jetbrains.kotlin.cli.common.ExitCode
import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
import org.junit.Assert
import org.junit.Ignore
import org.junit.Test
import java.io.File
import java.io.ObjectInputStream
import java.io.ObjectOutputStream
import java.io.*
import java.security.MessageDigest
import java.util.ArrayList
import kotlin.script.experimental.api.*
import kotlin.script.experimental.host.toScriptSource
import kotlin.script.experimental.host.with
@@ -24,13 +24,14 @@ import kotlin.script.experimental.jvm.util.classpathFromClass
import kotlin.script.experimental.jvmhost.BasicJvmScriptingHost
import kotlin.script.experimental.jvmhost.CompiledScriptJarsCache
import kotlin.script.experimental.jvmhost.JvmScriptCompiler
import kotlin.script.experimental.jvmhost.loadScriptFromJar
class CachingTest : TestCase() {
val simpleScript = "val x = 1\nprintln(\"x = \$x\")"
val simpleScript = "val x = 1\nprintln(\"x = \$x\")".toScriptSource()
val simpleScriptExpectedOutput = listOf("x = 1")
val scriptWithImport = "println(\"Hello from imported \$helloScriptName script!\")"
val scriptWithImport = "println(\"Hello from imported \$helloScriptName script!\")".toScriptSource()
val scriptWithImportExpectedOutput = listOf("Hello from helloWithVal script!", "Hello from imported helloWithVal script!")
@Test
@@ -117,16 +118,78 @@ class CachingTest : TestCase() {
compilationConfiguration = {
updateClasspath(classpathFromClass<ScriptingHostTest>()) // the class defined here should be in the classpath
implicitReceivers(Implicit::class)
},
evaluationConfiguration = {
implicitReceivers(Implicit)
}
) {
implicitReceivers(Implicit)
}
}
}
@Test
@Ignore // does not work reliably probably due to file cashing TODO: rewrite to more reliable variant
fun ignoredTestLocalDependencyWithJarCacheInvalidation() {
withTempDir("scriptingTestDepDir") { depDir ->
val standardJars = KotlinJars.kotlinScriptStandardJars
val outJar = File(depDir, "dependency.jar")
val inKt = File(depDir, "Dependency.kt").apply { writeText("class Dependency(val v: Int)") }
val outStream = ByteArrayOutputStream()
val compileExitCode = K2JVMCompiler().exec(
PrintStream(outStream),
"-d", outJar.path, "-no-stdlib", "-cp", standardJars.joinToString(File.pathSeparator), inKt.path
)
assertTrue(
"Compilation Failed:\n$outStream",
outStream.size() == 0 && compileExitCode == ExitCode.OK && outJar.exists()
)
withTempDir("scriptingTestJarChacheWithDep") { cacheDir ->
val cache = TestCompiledScriptJarsCache(cacheDir)
Assert.assertTrue(cache.baseDir.listFiles()!!.isEmpty())
val hostConfiguration = defaultJvmScriptingHostConfiguration.with {
jvm {
baseClassLoader.replaceOnlyDefault(null)
compilationCache(cache)
}
}
val host = BasicJvmScriptingHost(compiler = JvmScriptCompiler(hostConfiguration), evaluator = BasicJvmScriptEvaluator())
val scriptCompilationConfiguration = ScriptCompilationConfiguration {
updateClasspath(standardJars +outJar)
this.hostConfiguration.update { hostConfiguration }
}
val script = "Dependency(42).v".toScriptSource()
val res0 = host.eval(script, scriptCompilationConfiguration, null).valueOrThrow().returnValue
assertEquals(42, (res0 as? ResultValue.Value)?.value)
Assert.assertEquals(1, cache.storedScripts)
Assert.assertEquals(0, cache.retrievedScripts)
val res1 = host.eval(script, scriptCompilationConfiguration, null).valueOrThrow().returnValue
assertEquals(42, (res1 as? ResultValue.Value)?.value)
Assert.assertEquals(1, cache.storedScripts)
Assert.assertEquals(1, cache.retrievedScripts)
val outJar2 = File(depDir, "dependency2.jar")
outJar.renameTo(outJar2)
val cachedScriptJar = cache.baseDir.listFiles().single()
val loadedScript = cachedScriptJar.loadScriptFromJar(checkMissingDependencies = false)
val res2 = runBlocking {
BasicJvmScriptEvaluator().invoke(loadedScript!!)
}.valueOrThrow().returnValue
assertEquals("Dependency", (res2 as? ResultValue.Error)?.error?.message)
assertNull(cachedScriptJar.loadScriptFromJar(checkMissingDependencies = true))
assertNull(cache.get(script, scriptCompilationConfiguration))
assertEquals(0, cacheDir.listFiles().size)
}
}
}
private fun checkWithCache(
cache: ScriptingCacheWithCounters, script: String, expectedOutput: List<String>, checkDirectEval: Boolean = true,
cache: ScriptingCacheWithCounters, script: SourceCode, expectedOutput: List<String>, checkDirectEval: Boolean = true,
compilationConfiguration: ScriptCompilationConfiguration.Builder.() -> Unit = {},
evaluationConfiguration: ScriptEvaluationConfiguration.Builder.() -> Unit = {}
) {
@@ -151,7 +214,7 @@ class CachingTest : TestCase() {
var compiledScript: CompiledScript? = null
val output = captureOut {
runBlocking {
compiler(script.toScriptSource(), scriptCompilationConfiguration).onSuccess {
compiler(script, scriptCompilationConfiguration).onSuccess {
compiledScript = it
evaluator(it, scriptEvaluationConfiguration)
}.throwOnFailure()
@@ -162,7 +225,7 @@ class CachingTest : TestCase() {
Assert.assertEquals(0, cache.retrievedScripts)
if (checkDirectEval) {
val cachedScript = cache.get(script.toScriptSource(), scriptCompilationConfiguration)
val cachedScript = cache.get(script, scriptCompilationConfiguration)
Assert.assertNotNull(cachedScript)
Assert.assertEquals(1, cache.retrievedScripts)
@@ -184,7 +247,7 @@ class CachingTest : TestCase() {
}
val output3 = captureOut {
host.eval(script.toScriptSource(), scriptCompilationConfiguration, scriptEvaluationConfiguration).throwOnFailure()
host.eval(script, scriptCompilationConfiguration, scriptEvaluationConfiguration).throwOnFailure()
}.lines()
Assert.assertEquals(if (checkDirectEval) 2 else 1, cache.retrievedScripts)
Assert.assertEquals(output, output3)
libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/CompiledScriptJarsCache.kt
+7 -47
View File
@@ -6,15 +6,11 @@
package kotlin.script.experimental.jvmhost
import java.io.File
import java.net.URLClassLoader
import java.util.jar.JarInputStream
import kotlin.reflect.KClass
import kotlin.script.experimental.api.*
import kotlin.script.experimental.api.CompiledScript
import kotlin.script.experimental.api.ScriptCompilationConfiguration
import kotlin.script.experimental.api.SourceCode
import kotlin.script.experimental.jvm.CompiledJvmScriptsCache
import kotlin.script.experimental.jvm.baseClassLoader
import kotlin.script.experimental.jvm.impl.KJvmCompiledScript
import kotlin.script.experimental.jvm.impl.createScriptFromClassLoader
import kotlin.script.experimental.jvm.jvm
open class CompiledScriptJarsCache(val scriptToFile: (SourceCode, ScriptCompilationConfiguration) -> File?) :
CompiledJvmScriptsCache {
@@ -25,12 +21,11 @@ open class CompiledScriptJarsCache(val scriptToFile: (SourceCode, ScriptCompilat
if (!file.exists()) return null
val className = file.inputStream().use { ostr ->
JarInputStream(ostr).use {
it.manifest.mainAttributes.getValue("Main-Class")
}
return file.loadScriptFromJar() ?: run {
// invalidate cache if the script cannot be loaded
file.delete()
null
}
return KJvmCompiledScriptLazilyLoadedFromClasspath(className, listOf(file))
}
override fun store(
@@ -48,38 +43,3 @@ open class CompiledScriptJarsCache(val scriptToFile: (SourceCode, ScriptCompilat
}
}
private class KJvmCompiledScriptLazilyLoadedFromClasspath(
private val scriptClassFQName: String,
private val classPath: List<File>
) : CompiledScript {
private var loadedScript: KJvmCompiledScript? = null
fun getScriptOrError(): KJvmCompiledScript = loadedScript ?: throw RuntimeException("Compiled script is not loaded yet")
override suspend fun getClass(scriptEvaluationConfiguration: ScriptEvaluationConfiguration?): ResultWithDiagnostics<KClass<*>> {
if (loadedScript == null) {
val actualEvaluationConfiguration = scriptEvaluationConfiguration ?: ScriptEvaluationConfiguration()
val baseClassLoader = actualEvaluationConfiguration[ScriptEvaluationConfiguration.jvm.baseClassLoader]
val classLoader = URLClassLoader(
classPath.map { it.toURI().toURL() }.toTypedArray(),
baseClassLoader
)
loadedScript = createScriptFromClassLoader(scriptClassFQName, classLoader)
}
return getScriptOrError().getClass(scriptEvaluationConfiguration)
}
override val compilationConfiguration: ScriptCompilationConfiguration
get() = getScriptOrError().compilationConfiguration
override val sourceLocationId: String?
get() = getScriptOrError().sourceLocationId
override val otherScripts: List<CompiledScript>
get() = getScriptOrError().otherScripts
override val resultField: Pair<String, KotlinType>?
get() = getScriptOrError().resultField
}
libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/jvmScriptSaving.kt
+63 -1
View File
@@ -8,12 +8,18 @@ package kotlin.script.experimental.jvmhost
import org.jetbrains.kotlin.utils.KotlinPaths
import java.io.File
import java.io.FileOutputStream
import java.net.URI
import java.net.URLClassLoader
import java.util.jar.JarEntry
import java.util.jar.JarInputStream
import java.util.jar.JarOutputStream
import java.util.jar.Manifest
import kotlin.reflect.KClass
import kotlin.script.experimental.api.*
import kotlin.script.experimental.jvm.JvmDependency
import kotlin.script.experimental.jvm.baseClassLoader
import kotlin.script.experimental.jvm.impl.*
import kotlin.script.experimental.jvm.jvm
import kotlin.script.experimental.jvm.util.scriptCompilationClasspathFromContextOrNull
// TODO: generate execution code (main)
@@ -58,7 +64,8 @@ fun KJvmCompiledScript.saveToJar(outputJar: File) {
classLoader = this::class.java.classLoader,
wholeClasspath = false
) ?: emptyList()
val dependencies = (dependenciesFromScript + dependenciesForMain).distinct()
// saving only existing files, so the check for the existence in the loadScriptFromJar is meaningful
val dependencies = (dependenciesFromScript + dependenciesForMain).distinct().filter { it.exists() }
FileOutputStream(outputJar).use { fileStream ->
val manifest = Manifest()
manifest.mainAttributes.apply {
@@ -86,6 +93,26 @@ fun KJvmCompiledScript.saveToJar(outputJar: File) {
}
}
fun File.loadScriptFromJar(checkMissingDependencies: Boolean = true): CompiledScript? {
val (className: String?, classPathUrls) = this.inputStream().use { ostr ->
JarInputStream(ostr).use {
it.manifest.mainAttributes.getValue("Main-Class") to
(it.manifest.mainAttributes.getValue("Class-Path")?.split(" ") ?: emptyList())
}
}
if (className == null) return null
val classPath = classPathUrls.mapNotNullTo(mutableListOf(this)) { cpEntry ->
File(URI(cpEntry)).takeIf { it.exists() } ?: File(cpEntry).takeIf { it.exists() }
}
if (!checkMissingDependencies || classPathUrls.size + 1 == classPath.size) {
return KJvmCompiledScriptLazilyLoadedFromClasspath(className, classPath)
} else {
// Assuming that some script dependencies are not accessible anymore so the script is not valid and should be recompiled to reresolve dependencies
return null
}
}
open class BasicJvmScriptJarGenerator(val outputJar: File) : ScriptEvaluator {
override suspend operator fun invoke(
@@ -105,6 +132,41 @@ open class BasicJvmScriptJarGenerator(val outputJar: File) : ScriptEvaluator {
}
}
private class KJvmCompiledScriptLazilyLoadedFromClasspath(
private val scriptClassFQName: String,
private val classPath: List<File>
) : CompiledScript {
private var loadedScript: KJvmCompiledScript? = null
fun getScriptOrError(): KJvmCompiledScript = loadedScript ?: throw RuntimeException("Compiled script is not loaded yet")
override suspend fun getClass(scriptEvaluationConfiguration: ScriptEvaluationConfiguration?): ResultWithDiagnostics<KClass<*>> {
if (loadedScript == null) {
val actualEvaluationConfiguration = scriptEvaluationConfiguration ?: ScriptEvaluationConfiguration()
val baseClassLoader = actualEvaluationConfiguration[ScriptEvaluationConfiguration.jvm.baseClassLoader]
val classLoader = URLClassLoader(
classPath.map { it.toURI().toURL() }.toTypedArray(),
baseClassLoader
)
loadedScript = createScriptFromClassLoader(scriptClassFQName, classLoader)
}
return getScriptOrError().getClass(scriptEvaluationConfiguration)
}
override val compilationConfiguration: ScriptCompilationConfiguration
get() = getScriptOrError().compilationConfiguration
override val sourceLocationId: String?
get() = getScriptOrError().sourceLocationId
override val otherScripts: List<CompiledScript>
get() = getScriptOrError().otherScripts
override val resultField: Pair<String, KotlinType>?
get() = getScriptOrError().resultField
}
private fun failure(msg: String) =
ResultWithDiagnostics.Failure(msg.asErrorDiagnostics())
libraries/scripting/jvm/src/kotlin/script/experimental/jvm/impl/KJvmCompiledScript.kt
-2
View File
@@ -5,7 +5,6 @@
package kotlin.script.experimental.jvm.impl
import org.jetbrains.annotations.TestOnly
import java.io.*
import java.net.URL
import java.net.URLClassLoader
@@ -94,7 +93,6 @@ open class KJvmCompiledScript internal constructor(
)
}
@TestOnly
fun getCompiledModule() = compiledModule
private fun writeObject(outputStream: ObjectOutputStream) {