Scripting invalidate compiled cache if a dependency is missing

#KT-48303 fixed

libraries/scripting/jvm-host-test/test/kotlin/script/experimental/jvmhost/test/CachingTest.kt

@@ -7,13 +7,13 @@ package kotlin.script.experimental.jvmhost.test
 
 import junit.framework.TestCase
 import kotlinx.coroutines.runBlocking
+import org.jetbrains.kotlin.cli.common.ExitCode
+import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
 import org.junit.Assert
+import org.junit.Ignore
 import org.junit.Test
-import java.io.File
+import java.io.*
-import java.io.ObjectInputStream
-import java.io.ObjectOutputStream
 import java.security.MessageDigest
-import java.util.ArrayList
 import kotlin.script.experimental.api.*
 import kotlin.script.experimental.host.toScriptSource
 import kotlin.script.experimental.host.with
@@ -24,13 +24,14 @@ import kotlin.script.experimental.jvm.util.classpathFromClass
 import kotlin.script.experimental.jvmhost.BasicJvmScriptingHost
 import kotlin.script.experimental.jvmhost.CompiledScriptJarsCache
 import kotlin.script.experimental.jvmhost.JvmScriptCompiler
+import kotlin.script.experimental.jvmhost.loadScriptFromJar
 
 class CachingTest : TestCase() {
 
-    val simpleScript = "val x = 1\nprintln(\"x = \$x\")"
+    val simpleScript = "val x = 1\nprintln(\"x = \$x\")".toScriptSource()
     val simpleScriptExpectedOutput = listOf("x = 1")
 
-    val scriptWithImport = "println(\"Hello from imported \$helloScriptName script!\")"
+    val scriptWithImport = "println(\"Hello from imported \$helloScriptName script!\")".toScriptSource()
     val scriptWithImportExpectedOutput = listOf("Hello from helloWithVal script!", "Hello from imported helloWithVal script!")
 
     @Test
@@ -117,16 +118,78 @@ class CachingTest : TestCase() {
                 compilationConfiguration = {
                     updateClasspath(classpathFromClass<ScriptingHostTest>()) // the class defined here should be in the classpath
                     implicitReceivers(Implicit::class)
-                },
-                evaluationConfiguration = {
-                    implicitReceivers(Implicit)
                 }
+            ) {
+                implicitReceivers(Implicit)
+            }
+        }
+    }
+
+    @Test
+    @Ignore // does not work reliably probably due to file cashing TODO: rewrite to more reliable variant
+    fun ignoredTestLocalDependencyWithJarCacheInvalidation() {
+        withTempDir("scriptingTestDepDir") { depDir ->
+            val standardJars = KotlinJars.kotlinScriptStandardJars
+            val outJar = File(depDir, "dependency.jar")
+            val inKt = File(depDir, "Dependency.kt").apply { writeText("class Dependency(val v: Int)") }
+            val outStream = ByteArrayOutputStream()
+            val compileExitCode = K2JVMCompiler().exec(
+                PrintStream(outStream),
+                "-d", outJar.path, "-no-stdlib", "-cp", standardJars.joinToString(File.pathSeparator), inKt.path
             )
+            assertTrue(
+                "Compilation Failed:\n$outStream",
+                outStream.size() == 0 && compileExitCode == ExitCode.OK && outJar.exists()
+            )
+
+            withTempDir("scriptingTestJarChacheWithDep") { cacheDir ->
+                val cache = TestCompiledScriptJarsCache(cacheDir)
+                Assert.assertTrue(cache.baseDir.listFiles()!!.isEmpty())
+
+                val hostConfiguration = defaultJvmScriptingHostConfiguration.with {
+                    jvm {
+                        baseClassLoader.replaceOnlyDefault(null)
+                        compilationCache(cache)
+                    }
+                }
+                val host = BasicJvmScriptingHost(compiler = JvmScriptCompiler(hostConfiguration), evaluator = BasicJvmScriptEvaluator())
+
+                val scriptCompilationConfiguration = ScriptCompilationConfiguration {
+                    updateClasspath(standardJars +outJar)
+                    this.hostConfiguration.update { hostConfiguration }
+                }
+
+                val script = "Dependency(42).v".toScriptSource()
+
+                val res0 = host.eval(script, scriptCompilationConfiguration, null).valueOrThrow().returnValue
+                assertEquals(42, (res0 as? ResultValue.Value)?.value)
+                Assert.assertEquals(1, cache.storedScripts)
+                Assert.assertEquals(0, cache.retrievedScripts)
+
+                val res1 = host.eval(script, scriptCompilationConfiguration, null).valueOrThrow().returnValue
+                assertEquals(42, (res1 as? ResultValue.Value)?.value)
+                Assert.assertEquals(1, cache.storedScripts)
+                Assert.assertEquals(1, cache.retrievedScripts)
+
+                val outJar2 = File(depDir, "dependency2.jar")
+                outJar.renameTo(outJar2)
+
+                val cachedScriptJar = cache.baseDir.listFiles().single()
+                val loadedScript = cachedScriptJar.loadScriptFromJar(checkMissingDependencies = false)
+                val res2 = runBlocking {
+                    BasicJvmScriptEvaluator().invoke(loadedScript!!)
+                }.valueOrThrow().returnValue
+                assertEquals("Dependency", (res2 as? ResultValue.Error)?.error?.message)
+
+                assertNull(cachedScriptJar.loadScriptFromJar(checkMissingDependencies = true))
+                assertNull(cache.get(script, scriptCompilationConfiguration))
+                assertEquals(0, cacheDir.listFiles().size)
+            }
         }
     }
 
     private fun checkWithCache(
-        cache: ScriptingCacheWithCounters, script: String, expectedOutput: List<String>, checkDirectEval: Boolean = true,
+        cache: ScriptingCacheWithCounters, script: SourceCode, expectedOutput: List<String>, checkDirectEval: Boolean = true,
         compilationConfiguration: ScriptCompilationConfiguration.Builder.() -> Unit = {},
         evaluationConfiguration: ScriptEvaluationConfiguration.Builder.() -> Unit = {}
     ) {
@@ -151,7 +214,7 @@ class CachingTest : TestCase() {
         var compiledScript: CompiledScript? = null
         val output = captureOut {
             runBlocking {
-                compiler(script.toScriptSource(), scriptCompilationConfiguration).onSuccess {
+                compiler(script, scriptCompilationConfiguration).onSuccess {
                     compiledScript = it
                     evaluator(it, scriptEvaluationConfiguration)
                 }.throwOnFailure()
@@ -162,7 +225,7 @@ class CachingTest : TestCase() {
         Assert.assertEquals(0, cache.retrievedScripts)
 
         if (checkDirectEval) {
-            val cachedScript = cache.get(script.toScriptSource(), scriptCompilationConfiguration)
+            val cachedScript = cache.get(script, scriptCompilationConfiguration)
             Assert.assertNotNull(cachedScript)
             Assert.assertEquals(1, cache.retrievedScripts)
 
@@ -184,7 +247,7 @@ class CachingTest : TestCase() {
         }
 
         val output3 = captureOut {
-            host.eval(script.toScriptSource(), scriptCompilationConfiguration, scriptEvaluationConfiguration).throwOnFailure()
+            host.eval(script, scriptCompilationConfiguration, scriptEvaluationConfiguration).throwOnFailure()
         }.lines()
         Assert.assertEquals(if (checkDirectEval) 2 else 1, cache.retrievedScripts)
         Assert.assertEquals(output, output3)

libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/CompiledScriptJarsCache.kt

@@ -6,15 +6,11 @@
 package kotlin.script.experimental.jvmhost
 
 import java.io.File
-import java.net.URLClassLoader
+import kotlin.script.experimental.api.CompiledScript
-import java.util.jar.JarInputStream
+import kotlin.script.experimental.api.ScriptCompilationConfiguration
-import kotlin.reflect.KClass
+import kotlin.script.experimental.api.SourceCode
-import kotlin.script.experimental.api.*
 import kotlin.script.experimental.jvm.CompiledJvmScriptsCache
-import kotlin.script.experimental.jvm.baseClassLoader
 import kotlin.script.experimental.jvm.impl.KJvmCompiledScript
-import kotlin.script.experimental.jvm.impl.createScriptFromClassLoader
-import kotlin.script.experimental.jvm.jvm
 
 open class CompiledScriptJarsCache(val scriptToFile: (SourceCode, ScriptCompilationConfiguration) -> File?) :
     CompiledJvmScriptsCache {
@@ -25,12 +21,11 @@ open class CompiledScriptJarsCache(val scriptToFile: (SourceCode, ScriptCompilat
 
         if (!file.exists()) return null
 
-        val className = file.inputStream().use { ostr ->
+        return file.loadScriptFromJar() ?: run {
-            JarInputStream(ostr).use {
+            // invalidate cache if the script cannot be loaded
-                it.manifest.mainAttributes.getValue("Main-Class")
+            file.delete()
-            }
+            null
         }
-        return KJvmCompiledScriptLazilyLoadedFromClasspath(className, listOf(file))
     }
 
     override fun store(
@@ -48,38 +43,3 @@ open class CompiledScriptJarsCache(val scriptToFile: (SourceCode, ScriptCompilat
     }
 }
 
-private class KJvmCompiledScriptLazilyLoadedFromClasspath(
-    private val scriptClassFQName: String,
-    private val classPath: List<File>
-) : CompiledScript {
-
-    private var loadedScript: KJvmCompiledScript? = null
-
-    fun getScriptOrError(): KJvmCompiledScript = loadedScript ?: throw RuntimeException("Compiled script is not loaded yet")
-
-    override suspend fun getClass(scriptEvaluationConfiguration: ScriptEvaluationConfiguration?): ResultWithDiagnostics<KClass<*>> {
-        if (loadedScript == null) {
-            val actualEvaluationConfiguration = scriptEvaluationConfiguration ?: ScriptEvaluationConfiguration()
-            val baseClassLoader = actualEvaluationConfiguration[ScriptEvaluationConfiguration.jvm.baseClassLoader]
-            val classLoader = URLClassLoader(
-                classPath.map { it.toURI().toURL() }.toTypedArray(),
-                baseClassLoader
-            )
-            loadedScript = createScriptFromClassLoader(scriptClassFQName, classLoader)
-        }
-        return getScriptOrError().getClass(scriptEvaluationConfiguration)
-    }
-
-    override val compilationConfiguration: ScriptCompilationConfiguration
-        get() = getScriptOrError().compilationConfiguration
-
-    override val sourceLocationId: String?
-        get() = getScriptOrError().sourceLocationId
-
-    override val otherScripts: List<CompiledScript>
-        get() = getScriptOrError().otherScripts
-
-    override val resultField: Pair<String, KotlinType>?
-        get() = getScriptOrError().resultField
-}
-

libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/jvmScriptSaving.kt

@@ -8,12 +8,18 @@ package kotlin.script.experimental.jvmhost
 import org.jetbrains.kotlin.utils.KotlinPaths
 import java.io.File
 import java.io.FileOutputStream
+import java.net.URI
+import java.net.URLClassLoader
 import java.util.jar.JarEntry
+import java.util.jar.JarInputStream
 import java.util.jar.JarOutputStream
 import java.util.jar.Manifest
+import kotlin.reflect.KClass
 import kotlin.script.experimental.api.*
 import kotlin.script.experimental.jvm.JvmDependency
+import kotlin.script.experimental.jvm.baseClassLoader
 import kotlin.script.experimental.jvm.impl.*
+import kotlin.script.experimental.jvm.jvm
 import kotlin.script.experimental.jvm.util.scriptCompilationClasspathFromContextOrNull
 
 // TODO: generate execution code (main)
@@ -58,7 +64,8 @@ fun KJvmCompiledScript.saveToJar(outputJar: File) {
         classLoader = this::class.java.classLoader,
         wholeClasspath = false
     ) ?: emptyList()
-    val dependencies = (dependenciesFromScript + dependenciesForMain).distinct()
+    // saving only existing files, so the check for the existence in the loadScriptFromJar is meaningful
+    val dependencies = (dependenciesFromScript + dependenciesForMain).distinct().filter { it.exists() }
     FileOutputStream(outputJar).use { fileStream ->
         val manifest = Manifest()
         manifest.mainAttributes.apply {
@@ -86,6 +93,26 @@ fun KJvmCompiledScript.saveToJar(outputJar: File) {
     }
 }
 
+fun File.loadScriptFromJar(checkMissingDependencies: Boolean = true): CompiledScript? {
+    val (className: String?, classPathUrls) = this.inputStream().use { ostr ->
+        JarInputStream(ostr).use {
+            it.manifest.mainAttributes.getValue("Main-Class") to
+                    (it.manifest.mainAttributes.getValue("Class-Path")?.split(" ") ?: emptyList())
+        }
+    }
+    if (className == null) return null
+
+    val classPath = classPathUrls.mapNotNullTo(mutableListOf(this)) { cpEntry ->
+        File(URI(cpEntry)).takeIf { it.exists() } ?: File(cpEntry).takeIf { it.exists() }
+    }
+    if (!checkMissingDependencies || classPathUrls.size + 1 == classPath.size) {
+        return KJvmCompiledScriptLazilyLoadedFromClasspath(className, classPath)
+    } else {
+        // Assuming that some script dependencies are not accessible anymore so the script is not valid and should be recompiled to reresolve dependencies
+        return null
+    }
+}
+
 open class BasicJvmScriptJarGenerator(val outputJar: File) : ScriptEvaluator {
 
     override suspend operator fun invoke(
@@ -105,6 +132,41 @@ open class BasicJvmScriptJarGenerator(val outputJar: File) : ScriptEvaluator {
     }
 }
 
+private class KJvmCompiledScriptLazilyLoadedFromClasspath(
+    private val scriptClassFQName: String,
+    private val classPath: List<File>
+) : CompiledScript {
+
+    private var loadedScript: KJvmCompiledScript? = null
+
+    fun getScriptOrError(): KJvmCompiledScript = loadedScript ?: throw RuntimeException("Compiled script is not loaded yet")
+
+    override suspend fun getClass(scriptEvaluationConfiguration: ScriptEvaluationConfiguration?): ResultWithDiagnostics<KClass<*>> {
+        if (loadedScript == null) {
+            val actualEvaluationConfiguration = scriptEvaluationConfiguration ?: ScriptEvaluationConfiguration()
+            val baseClassLoader = actualEvaluationConfiguration[ScriptEvaluationConfiguration.jvm.baseClassLoader]
+            val classLoader = URLClassLoader(
+                classPath.map { it.toURI().toURL() }.toTypedArray(),
+                baseClassLoader
+            )
+            loadedScript = createScriptFromClassLoader(scriptClassFQName, classLoader)
+        }
+        return getScriptOrError().getClass(scriptEvaluationConfiguration)
+    }
+
+    override val compilationConfiguration: ScriptCompilationConfiguration
+        get() = getScriptOrError().compilationConfiguration
+
+    override val sourceLocationId: String?
+        get() = getScriptOrError().sourceLocationId
+
+    override val otherScripts: List<CompiledScript>
+        get() = getScriptOrError().otherScripts
+
+    override val resultField: Pair<String, KotlinType>?
+        get() = getScriptOrError().resultField
+}
+
 private fun failure(msg: String) =
     ResultWithDiagnostics.Failure(msg.asErrorDiagnostics())
 

libraries/scripting/jvm/src/kotlin/script/experimental/jvm/impl/KJvmCompiledScript.kt

@@ -5,7 +5,6 @@
 
 package kotlin.script.experimental.jvm.impl
 
-import org.jetbrains.annotations.TestOnly
 import java.io.*
 import java.net.URL
 import java.net.URLClassLoader
@@ -94,7 +93,6 @@ open class KJvmCompiledScript internal constructor(
         )
     }
 
-    @TestOnly
     fun getCompiledModule() = compiledModule
 
     private fun writeObject(outputStream: ObjectOutputStream) {