Implement base zip cache with tests...

plus some relevant fixes and refactorings
2019-07-24 18:47:55 +02:00
parent c7eb9e9520
commit 42dde50b6b
6 changed files with 164 additions and 23 deletions
@@ -81,6 +81,8 @@ private fun ScriptCompilationConfiguration.Builder.propertiesFromTemplate(
 private val KClass<*>.kotlinScriptAnnotation: KotlinScript
    get() = findAnnotation()
        ?: when (this@kotlinScriptAnnotation.qualifiedName) {
+            // Any is the default template, so use a default annotation
+            Any::class.qualifiedName,
            // transitions to the new scripting API: substituting annotations for standard templates from script-runtime
            "$SCRIPT_RUNTIME_TEMPLATES_PACKAGE.SimpleScriptTemplate",
            "$SCRIPT_RUNTIME_TEMPLATES_PACKAGE.ScriptTemplateWithArgs",
@@ -31,6 +31,17 @@ class ScriptingHostConfiguration(baseScriptingConfigurations: Iterable<Scripting
    companion object : ScriptingHostConfigurationKeys
 }

+/**
+ * An alternative to the constructor with base configuration, which returns a new configuration only if [body] adds anything
+ * to the original one, otherwise returns original
+ */
+fun ScriptingHostConfiguration?.with(body: ScriptingHostConfiguration.Builder.() -> Unit): ScriptingHostConfiguration {
+    val newConfiguration =
+        if (this == null) ScriptingHostConfiguration(body = body)
+        else ScriptingHostConfiguration(this, body = body)
+    return if (newConfiguration != this) newConfiguration else this
+}
+
 /**
 * The list of all dependencies required for the script base class and refinement callbacks
 */
@@ -16,11 +16,14 @@ import java.nio.file.Files
 import java.security.MessageDigest
 import kotlin.script.experimental.api.*
 import kotlin.script.experimental.host.toScriptSource
-import kotlin.script.experimental.jvm.BasicJvmScriptEvaluator
-import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration
+import kotlin.script.experimental.host.with
+import kotlin.script.experimental.jvm.*
 import kotlin.script.experimental.jvm.impl.KJvmCompiledScript
-import kotlin.script.experimental.jvmhost.*
-import kotlin.script.templates.standard.SimpleScriptTemplate
+import kotlin.script.experimental.jvm.util.KotlinJars
+import kotlin.script.experimental.jvmhost.BasicJvmScriptingHost
+import kotlin.script.experimental.jvmhost.CompiledJvmScriptsCache
+import kotlin.script.experimental.jvmhost.CompiledScriptJarsCache
+import kotlin.script.experimental.jvmhost.JvmScriptCompiler

 class CachingTest : TestCase() {

@@ -63,16 +66,53 @@ class CachingTest : TestCase() {
        }
    }

+    @Test
+    fun testJarCache() {
+        withTempDir("scriptingTestJarCache") { cacheDir ->
+            val cache = TestCompiledScriptJarsCache(cacheDir)
+            Assert.assertTrue(cache.baseDir.listFiles()!!.isEmpty())
+
+            checkWithCache(cache, simpleScript, simpleScriptExpectedOutput)
+
+            val scriptOut = runScriptFromJar(cache.baseDir.listFiles()!!.first { it.extension == "jar" })
+
+            Assert.assertEquals(simpleScriptExpectedOutput, scriptOut)
+        }
+    }
+
+    @Test
+    fun testSimpleImportWithJarCache() {
+        withTempDir("scriptingTestJarCache") { cacheDir ->
+            val cache = TestCompiledScriptJarsCache(cacheDir)
+            Assert.assertTrue(cache.baseDir.listFiles()!!.isEmpty())
+
+            checkWithCache(cache, scriptWithImport, scriptWithImportExpectedOutput) { makeSimpleConfigurationWithTestImport() }
+
+            // cannot make it work in this form - it requires a dependency on the current test classes, but classes directory seems
+            // not work when specified in the manifest
+            // TODO: find a way to make it work
+//            val scriptOut = runScriptFromJar(cache.baseDir.listFiles()!!.first { it.extension == "jar" })
+//
+//            Assert.assertEquals(scriptWithImportExpectedOutput, scriptOut)
+        }
+    }
+
    private fun checkWithCache(
        cache: ScriptingCacheWithCounters, script: String, expectedOutput: List<String>,
        configurationBuilder: ScriptCompilationConfiguration.Builder.() -> Unit = {}
    ) {
-        val compiler = JvmScriptCompiler(defaultJvmScriptingHostConfiguration, cache = cache)
+        val hostConfiguration = defaultJvmScriptingHostConfiguration.with {
+            jvm {
+                baseClassLoader.replaceOnlyDefault(null)
+            }
+        }
+        val compiler = JvmScriptCompiler(hostConfiguration, cache = cache)
        val evaluator = BasicJvmScriptEvaluator()
        val host = BasicJvmScriptingHost(compiler = compiler, evaluator = evaluator)

-        val scriptCompilationConfiguration =
-            createJvmCompilationConfigurationFromTemplate<SimpleScriptTemplate>(body = configurationBuilder)
+        val scriptCompilationConfiguration = ScriptCompilationConfiguration(body = configurationBuilder).with {
+            updateClasspath(KotlinJars.kotlinScriptStandardJarsWithReflect)
+        }

        Assert.assertEquals(0, cache.storedScripts)
        var compiledScript: CompiledScript<*>? = null
@@ -95,11 +135,11 @@ class CachingTest : TestCase() {
        val compiledScriptClassRes = runBlocking { compiledScript!!.getClass(null) }
        val cachedScriptClassRes = runBlocking { cachedScript!!.getClass(null) }

-        val compiledScriptClass = compiledScriptClassRes.valueOrNull()
-        val cachedScriptClass = cachedScriptClassRes.valueOrNull()
+        val compiledScriptClass = compiledScriptClassRes.valueOrThrow()
+        val cachedScriptClass = cachedScriptClassRes.valueOrThrow()

-        Assert.assertEquals(compiledScriptClass!!.qualifiedName, cachedScriptClass!!.qualifiedName)
-        Assert.assertEquals(compiledScriptClass!!.supertypes, cachedScriptClass!!.supertypes)
+        Assert.assertEquals(compiledScriptClass.qualifiedName, cachedScriptClass.qualifiedName)
+        Assert.assertEquals(compiledScriptClass.supertypes, cachedScriptClass.supertypes)

        val output2 = captureOut {
            runBlocking {
@@ -108,7 +148,9 @@ class CachingTest : TestCase() {
        }.lines()
        Assert.assertEquals(output, output2)

-        val output3 = captureOut { evalScriptWithConfiguration(script, host, configurationBuilder).throwOnFailure() }.lines()
+        val output3 = captureOut {
+            host.eval(script.toScriptSource(), scriptCompilationConfiguration, null).throwOnFailure()
+        }.lines()
        Assert.assertEquals(2, cache.retrievedScripts)
        Assert.assertEquals(output, output3)
    }
@@ -174,6 +216,29 @@ private class FileBasedScriptCache(val baseDir: File) : ScriptingCacheWithCounte
        private set
 }

+class TestCompiledScriptJarsCache(val baseDir: File) : CompiledScriptJarsCache(
+    { script, scriptCompilationConfiguration ->
+        File(baseDir, uniqueScriptHash(script, scriptCompilationConfiguration) + ".jar")
+    }), ScriptingCacheWithCounters
+{
+    override fun get(script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration): CompiledScript<*>? =
+        super.get(script, scriptCompilationConfiguration)?.also { retrievedScripts++ }
+
+    override fun store(
+        compiledScript: CompiledScript<*>,
+        script: SourceCode,
+        scriptCompilationConfiguration: ScriptCompilationConfiguration
+    ) {
+        super.store(compiledScript, script, scriptCompilationConfiguration).also { storedScripts++ }
+    }
+
+    override var storedScripts: Int = 0
+        private set
+
+    override var retrievedScripts: Int = 0
+        private set
+}
+
 internal fun uniqueScriptHash(script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration): String {
    val digestWrapper = MessageDigest.getInstance("MD5")
    digestWrapper.update(script.text.toByteArray())
@@ -14,8 +14,9 @@ import org.jetbrains.org.objectweb.asm.ClassVisitor
 import org.jetbrains.org.objectweb.asm.Opcodes
 import org.junit.Assert
 import org.junit.Test
-import java.io.*
-import java.lang.RuntimeException
+import java.io.ByteArrayOutputStream
+import java.io.File
+import java.io.PrintStream
 import java.net.URLClassLoader
 import java.nio.file.Files
 import java.util.concurrent.TimeUnit
@@ -161,7 +162,7 @@ class ScriptingHostTest : TestCase() {
        checkInvokeMain(Thread.currentThread().contextClassLoader)

        val outputFromProcess = runScriptFromJar(outJar)
-        Assert.assertEquals(greeting, outputFromProcess)
+        Assert.assertEquals(listOf(greeting), outputFromProcess)
    }

    @Test
@@ -373,12 +374,12 @@ class ScriptingHostTest : TestCase() {
    }
 }

-internal fun runScriptFromJar(jar: File): String {
+internal fun runScriptFromJar(jar: File): List<String> {
    val javaExecutable = File(File(System.getProperty("java.home"), "bin"), "java")
    val args = listOf(javaExecutable.absolutePath, "-jar", jar.path)
    val processBuilder = ProcessBuilder(args)
    processBuilder.redirectErrorStream(true)
-    return run {
+    val r = run {
        val process = processBuilder.start()
        process.waitFor(10, TimeUnit.SECONDS)
        val out = process.inputStream.reader().readText()
@@ -389,6 +390,7 @@ internal fun runScriptFromJar(jar: File): String {
            out
        }
    }.trim()
+    return r.lineSequence().map { it.trim() }.toList()
 }

 fun <T> ResultWithDiagnostics<T>.throwOnFailure(): ResultWithDiagnostics<T> = apply {
@@ -421,6 +423,7 @@ internal fun evalScriptWithConfiguration(
 }

 internal fun ScriptCompilationConfiguration.Builder.makeSimpleConfigurationWithTestImport() {
+    updateClasspath(classpathFromClass<ScriptingHostTest>()) // the lambda below should be in the classpath
    refineConfiguration {
        beforeCompiling { ctx ->
            val importedScript = File(ScriptingHostTest.TEST_DATA_DIR, "importTest/helloWithVal.kts")
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.script.experimental.jvmhost
+
+import java.io.File
+import java.net.URLClassLoader
+import java.util.jar.JarInputStream
+import kotlin.script.experimental.api.CompiledScript
+import kotlin.script.experimental.api.ScriptCompilationConfiguration
+import kotlin.script.experimental.api.SourceCode
+import kotlin.script.experimental.api.hostConfiguration
+import kotlin.script.experimental.host.ScriptingHostConfiguration
+import kotlin.script.experimental.jvm.baseClassLoader
+import kotlin.script.experimental.jvm.impl.KJvmCompiledScript
+import kotlin.script.experimental.jvm.impl.createScriptFromClassLoader
+import kotlin.script.experimental.jvm.jvm
+
+open class CompiledScriptJarsCache(val scriptToFile: (SourceCode, ScriptCompilationConfiguration) -> File?) : CompiledJvmScriptsCache {
+
+    override fun get(script: SourceCode, scriptCompilationConfiguration: ScriptCompilationConfiguration): CompiledScript<*>? {
+        val file = scriptToFile(script, scriptCompilationConfiguration)
+            ?: throw IllegalArgumentException("Unable to find a mapping to a file for the script $script")
+
+        if (!file.exists()) return null
+
+        val className = file.inputStream().use { ostr ->
+            JarInputStream(ostr).use {
+                it.manifest.mainAttributes.getValue("Main-Class")
+            }
+        }
+        val classLoader = URLClassLoader(
+            arrayOf(file.toURI().toURL()),
+            scriptCompilationConfiguration[ScriptCompilationConfiguration.hostConfiguration]?.get(ScriptingHostConfiguration.jvm.baseClassLoader)
+        )
+
+        return createScriptFromClassLoader(className, classLoader)
+    }
+
+    override fun store(
+        compiledScript: CompiledScript<*>,
+        script: SourceCode,
+        scriptCompilationConfiguration: ScriptCompilationConfiguration
+    ) {
+        val file = scriptToFile(script, scriptCompilationConfiguration)
+            ?: throw IllegalArgumentException("Unable to find a mapping to a file for the script $script")
+
+        val jvmScript = (compiledScript as? KJvmCompiledScript<*>)
+            ?: throw IllegalArgumentException("Unsupported script type ${compiledScript::class.java.name}")
+
+        jvmScript.saveToJar(file)
+    }
+}
@@ -43,19 +43,24 @@ open class JvmScriptCompiler(
    override suspend operator fun invoke(
        script: SourceCode,
        scriptCompilationConfiguration: ScriptCompilationConfiguration
-    ): ResultWithDiagnostics<CompiledScript<*>> =
-        scriptCompilationConfiguration.with {
+    ): ResultWithDiagnostics<CompiledScript<*>> {
+
+        // TODO: implement caching deeper in the compilation pipeline - the actual configuration should be calculated first, with dependencies and imported scripts
+        // Note that previous implementation wasn't a solution for that and added problems with cache usage. Now it is consistent although shallow.
+
+        val cached = cache.get(script, scriptCompilationConfiguration)
+        if (cached != null) return cached.asSuccess()
+
+        return scriptCompilationConfiguration.with {
            hostConfiguration(this@JvmScriptCompiler.hostConfiguration)
        }.refineBeforeParsing(script).onSuccess { refinedConfiguration ->
-            val cached = cache.get(script, refinedConfiguration)
-
-            if (cached != null) return cached.asSuccess()

            compilerProxy.compile(script, refinedConfiguration).also {
                if (it is ResultWithDiagnostics.Success) {
-                    cache.store(it.value, script, refinedConfiguration)
+                    cache.store(it.value, script, scriptCompilationConfiguration)
                }
            }
        }
+    }
 }