Implement base zip cache with tests...

plus some relevant fixes and refactorings

libraries/scripting/common/src/kotlin/script/experimental/host/configurationFromTemplate.kt

@@ -81,6 +81,8 @@ private fun ScriptCompilationConfiguration.Builder.propertiesFromTemplate(
 private val KClass<*>.kotlinScriptAnnotation: KotlinScript
     get() = findAnnotation()
         ?: when (this@kotlinScriptAnnotation.qualifiedName) {
+            // Any is the default template, so use a default annotation
+            Any::class.qualifiedName,
             // transitions to the new scripting API: substituting annotations for standard templates from script-runtime
             "$SCRIPT_RUNTIME_TEMPLATES_PACKAGE.SimpleScriptTemplate",
             "$SCRIPT_RUNTIME_TEMPLATES_PACKAGE.ScriptTemplateWithArgs",

libraries/scripting/common/src/kotlin/script/experimental/host/hostConfiguration.kt

@@ -31,6 +31,17 @@ class ScriptingHostConfiguration(baseScriptingConfigurations: Iterable<Scripting
     companion object : ScriptingHostConfigurationKeys
 }
 
+/**
+ * An alternative to the constructor with base configuration, which returns a new configuration only if [body] adds anything
+ * to the original one, otherwise returns original
+ */
+fun ScriptingHostConfiguration?.with(body: ScriptingHostConfiguration.Builder.() -> Unit): ScriptingHostConfiguration {
+    val newConfiguration =
+        if (this == null) ScriptingHostConfiguration(body = body)
+        else ScriptingHostConfiguration(this, body = body)
+    return if (newConfiguration != this) newConfiguration else this
+}
+
 /**
  * The list of all dependencies required for the script base class and refinement callbacks
  */

libraries/scripting/jvm-host-test/test/kotlin/script/experimental/jvmhost/test/CachingTest.kt

@@ -16,11 +16,14 @@ import java.nio.file.Files
 import java.security.MessageDigest
 import kotlin.script.experimental.api.*
 import kotlin.script.experimental.host.toScriptSource
-import kotlin.script.experimental.jvm.BasicJvmScriptEvaluator
+import kotlin.script.experimental.host.with
-import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration
+import kotlin.script.experimental.jvm.*
 import kotlin.script.experimental.jvm.impl.KJvmCompiledScript
-import kotlin.script.experimental.jvmhost.*
+import kotlin.script.experimental.jvm.util.KotlinJars
-import kotlin.script.templates.standard.SimpleScriptTemplate
+import kotlin.script.experimental.jvmhost.BasicJvmScriptingHost
+import kotlin.script.experimental.jvmhost.CompiledJvmScriptsCache
+import kotlin.script.experimental.jvmhost.CompiledScriptJarsCache
+import kotlin.script.experimental.jvmhost.JvmScriptCompiler
 
 class CachingTest : TestCase() {
 
@@ -63,16 +66,53 @@ class CachingTest : TestCase() {
         }
     }
 
+    @Test
+    fun testJarCache() {
+        withTempDir("scriptingTestJarCache") { cacheDir ->
+            val cache = TestCompiledScriptJarsCache(cacheDir)
+            Assert.assertTrue(cache.baseDir.listFiles()!!.isEmpty())
+
+            checkWithCache(cache, simpleScript, simpleScriptExpectedOutput)
+
+            val scriptOut = runScriptFromJar(cache.baseDir.listFiles()!!.first { it.extension == "jar" })
+
+            Assert.assertEquals(simpleScriptExpectedOutput, scriptOut)
+        }
+    }
+
+    @Test
+    fun testSimpleImportWithJarCache() {
+        withTempDir("scriptingTestJarCache") { cacheDir ->
+            val cache = TestCompiledScriptJarsCache(cacheDir)
+            Assert.assertTrue(cache.baseDir.listFiles()!!.isEmpty())
+
+            checkWithCache(cache, scriptWithImport, scriptWithImportExpectedOutput) { makeSimpleConfigurationWithTestImport() }
+
+            // cannot make it work in this form - it requires a dependency on the current test classes, but classes directory seems
+            // not work when specified in the manifest
+            // TODO: find a way to make it work
+//            val scriptOut = runScriptFromJar(cache.baseDir.listFiles()!!.first { it.extension == "jar" })
+//
+//            Assert.assertEquals(scriptWithImportExpectedOutput, scriptOut)
+        }
+    }
+
     private fun checkWithCache(
         cache: ScriptingCacheWithCounters, script: String, expectedOutput: List<String>,
         configurationBuilder: ScriptCompilationConfiguration.Builder.() -> Unit = {}
     ) {
-        val compiler = JvmScriptCompiler(defaultJvmScriptingHostConfiguration, cache = cache)
+        val hostConfiguration = defaultJvmScriptingHostConfiguration.with {
+            jvm {
+                baseClassLoader.replaceOnlyDefault(null)
+            }
+        }
+        val compiler = JvmScriptCompiler(hostConfiguration, cache = cache)
         val evaluator = BasicJvmScriptEvaluator()
         val host = BasicJvmScriptingHost(compiler = compiler, evaluator = evaluator)
 
-        val scriptCompilationConfiguration =
+        val scriptCompilationConfiguration = ScriptCompilationConfiguration(body = configurationBuilder).with {
-            createJvmCompilationConfigurationFromTemplate<SimpleScriptTemplate>(body = configurationBuilder)
+            updateClasspath(KotlinJars.kotlinScriptStandardJarsWithReflect)
+        }
 
         Assert.assertEquals(0, cache.storedScripts)
         var compiledScript: CompiledScript<*>? = null
@@ -95,11 +135,11 @@ class CachingTest : TestCase() {
         val compiledScriptClassRes = runBlocking { compiledScript!!.getClass(null) }
         val cachedScriptClassRes = runBlocking { cachedScript!!.getClass(null) }
 
-        val compiledScriptClass = compiledScriptClassRes.valueOrNull()
+        val compiledScriptClass = compiledScriptClassRes.valueOrThrow()
-        val cachedScriptClass = cachedScriptClassRes.valueOrNull()
+        val cachedScriptClass = cachedScriptClassRes.valueOrThrow()
 
-        Assert.assertEquals(compiledScriptClass!!.qualifiedName, cachedScriptClass!!.qualifiedName)
+        Assert.assertEquals(compiledScriptClass.qualifiedName, cachedScriptClass.qualifiedName)
-        Assert.assertEquals(compiledScriptClass!!.supertypes, cachedScriptClass!!.supertypes)
+        Assert.assertEquals(compiledScriptClass.supertypes, cachedScriptClass.supertypes)
 
         val output2 = captureOut {
             runBlocking {
@@ -108,7 +148,9 @@ class CachingTest : TestCase() {
         }.lines()
         Assert.assertEquals(output, output2)
 
-        val output3 = captureOut { evalScriptWithConfiguration(script, host, configurationBuilder).throwOnFailure() }.lines()
+        val output3 = captureOut {
+            host.eval(script.toScriptSource(), scriptCompilationConfiguration, null).throwOnFailure()
+        }.lines()
         Assert.assertEquals(2, cache.retrievedScripts)
         Assert.assertEquals(output, output3)
     }
@@ -174,6 +216,29 @@ private class FileBasedScriptCache(val baseDir: File) : ScriptingCacheWithCounte
         private set
 }
 
+class TestCompiledScriptJarsCache(val baseDir: File) : CompiledScriptJarsCache(
+    { script, scriptCompilationConfiguration ->
+        File(baseDir, uniqueScriptHash(script, scriptCompilationConfiguration) + ".jar")
+    }), ScriptingCacheWithCounters
+{
+    override fun get(script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration): CompiledScript<*>? =
+        super.get(script, scriptCompilationConfiguration)?.also { retrievedScripts++ }
+
+    override fun store(
+        compiledScript: CompiledScript<*>,
+        script: SourceCode,
+        scriptCompilationConfiguration: ScriptCompilationConfiguration
+    ) {
+        super.store(compiledScript, script, scriptCompilationConfiguration).also { storedScripts++ }
+    }
+
+    override var storedScripts: Int = 0
+        private set
+
+    override var retrievedScripts: Int = 0
+        private set
+}
+
 internal fun uniqueScriptHash(script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration): String {
     val digestWrapper = MessageDigest.getInstance("MD5")
     digestWrapper.update(script.text.toByteArray())

libraries/scripting/jvm-host-test/test/kotlin/script/experimental/jvmhost/test/ScriptingHostTest.kt

@@ -14,8 +14,9 @@ import org.jetbrains.org.objectweb.asm.ClassVisitor
 import org.jetbrains.org.objectweb.asm.Opcodes
 import org.junit.Assert
 import org.junit.Test
-import java.io.*
+import java.io.ByteArrayOutputStream
-import java.lang.RuntimeException
+import java.io.File
+import java.io.PrintStream
 import java.net.URLClassLoader
 import java.nio.file.Files
 import java.util.concurrent.TimeUnit
@@ -161,7 +162,7 @@ class ScriptingHostTest : TestCase() {
         checkInvokeMain(Thread.currentThread().contextClassLoader)
 
         val outputFromProcess = runScriptFromJar(outJar)
-        Assert.assertEquals(greeting, outputFromProcess)
+        Assert.assertEquals(listOf(greeting), outputFromProcess)
     }
 
     @Test
@@ -373,12 +374,12 @@ class ScriptingHostTest : TestCase() {
     }
 }
 
-internal fun runScriptFromJar(jar: File): String {
+internal fun runScriptFromJar(jar: File): List<String> {
     val javaExecutable = File(File(System.getProperty("java.home"), "bin"), "java")
     val args = listOf(javaExecutable.absolutePath, "-jar", jar.path)
     val processBuilder = ProcessBuilder(args)
     processBuilder.redirectErrorStream(true)
-    return run {
+    val r = run {
         val process = processBuilder.start()
         process.waitFor(10, TimeUnit.SECONDS)
         val out = process.inputStream.reader().readText()
@@ -389,6 +390,7 @@ internal fun runScriptFromJar(jar: File): String {
             out
         }
     }.trim()
+    return r.lineSequence().map { it.trim() }.toList()
 }
 
 fun <T> ResultWithDiagnostics<T>.throwOnFailure(): ResultWithDiagnostics<T> = apply {
@@ -421,6 +423,7 @@ internal fun evalScriptWithConfiguration(
 }
 
 internal fun ScriptCompilationConfiguration.Builder.makeSimpleConfigurationWithTestImport() {
+    updateClasspath(classpathFromClass<ScriptingHostTest>()) // the lambda below should be in the classpath
     refineConfiguration {
         beforeCompiling { ctx ->
             val importedScript = File(ScriptingHostTest.TEST_DATA_DIR, "importTest/helloWithVal.kts")

libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/jvmScriptCaching.kt

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.script.experimental.jvmhost
+
+import java.io.File
+import java.net.URLClassLoader
+import java.util.jar.JarInputStream
+import kotlin.script.experimental.api.CompiledScript
+import kotlin.script.experimental.api.ScriptCompilationConfiguration
+import kotlin.script.experimental.api.SourceCode
+import kotlin.script.experimental.api.hostConfiguration
+import kotlin.script.experimental.host.ScriptingHostConfiguration
+import kotlin.script.experimental.jvm.baseClassLoader
+import kotlin.script.experimental.jvm.impl.KJvmCompiledScript
+import kotlin.script.experimental.jvm.impl.createScriptFromClassLoader
+import kotlin.script.experimental.jvm.jvm
+
+open class CompiledScriptJarsCache(val scriptToFile: (SourceCode, ScriptCompilationConfiguration) -> File?) : CompiledJvmScriptsCache {
+
+    override fun get(script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration): CompiledScript<*>? {
+        val file = scriptToFile(script, scriptCompilationConfiguration)
+            ?: throw IllegalArgumentException("Unable to find a mapping to a file for the script $script")
+
+        if (!file.exists()) return null
+
+        val className = file.inputStream().use { ostr ->
+            JarInputStream(ostr).use {
+                it.manifest.mainAttributes.getValue("Main-Class")
+            }
+        }
+        val classLoader = URLClassLoader(
+            arrayOf(file.toURI().toURL()),
+            scriptCompilationConfiguration[ScriptCompilationConfiguration.hostConfiguration]?.get(ScriptingHostConfiguration.jvm.baseClassLoader)
+        )
+
+        return createScriptFromClassLoader(className, classLoader)
+    }
+
+    override fun store(
+        compiledScript: CompiledScript<*>,
+        script: SourceCode,
+        scriptCompilationConfiguration: ScriptCompilationConfiguration
+    ) {
+        val file = scriptToFile(script, scriptCompilationConfiguration)
+            ?: throw IllegalArgumentException("Unable to find a mapping to a file for the script $script")
+
+        val jvmScript = (compiledScript as? KJvmCompiledScript<*>)
+            ?: throw IllegalArgumentException("Unsupported script type ${compiledScript::class.java.name}")
+
+        jvmScript.saveToJar(file)
+    }
+}

libraries/scripting/jvm-host/src/kotlin/script/experimental/jvmhost/jvmScriptCompilation.kt

@@ -43,19 +43,24 @@ open class JvmScriptCompiler(
     override suspend operator fun invoke(
         script: SourceCode,
         scriptCompilationConfiguration: ScriptCompilationConfiguration
-    ): ResultWithDiagnostics<CompiledScript<*>> =
+    ): ResultWithDiagnostics<CompiledScript<*>> {
-        scriptCompilationConfiguration.with {
+
+        // TODO: implement caching deeper in the compilation pipeline - the actual configuration should be calculated first, with dependencies and imported scripts
+        // Note that previous implementation wasn't a solution for that and added problems with cache usage. Now it is consistent although shallow.
+
+        val cached = cache.get(script, scriptCompilationConfiguration)
+        if (cached != null) return cached.asSuccess()
+
+        return scriptCompilationConfiguration.with {
             hostConfiguration(this@JvmScriptCompiler.hostConfiguration)
         }.refineBeforeParsing(script).onSuccess { refinedConfiguration ->
-            val cached = cache.get(script, refinedConfiguration)
-
-            if (cached != null) return cached.asSuccess()
 
             compilerProxy.compile(script, refinedConfiguration).also {
                 if (it is ResultWithDiagnostics.Success) {
-                    cache.store(it.value, script, refinedConfiguration)
+                    cache.store(it.value, script, scriptCompilationConfiguration)
                 }
             }
         }
+    }
 }