Implement resolve top-level functions and props from classloader

#KT-33892 fixed
2019-09-30 15:50:43 +02:00
parent 7b93970ce2
commit 2219b950f1
13 changed files with 461 additions and 93 deletions
@@ -52,10 +52,9 @@ class ResolveDependenciesTest : TestCase() {
        runScriptAndCheckResult(classImportScript, configurationWithDependenciesFromClasspath, null, 42)
    }

-    @Ignore
    @Test
    // This doesn't work since there is no way to resolve a top-level function/property via reflection now (see #KT-33892)
-    fun ignore_testResolveFunAndValFromClassloader() {
+    fun testResolveFunAndValFromClassloader() {
        runScriptAndCheckResult(funAndValAccessScript, configurationWithDependenciesFromClassloader, null, 42)
        runScriptAndCheckResult(funAndValImportScript, configurationWithDependenciesFromClassloader, null, 42)
    }
@@ -10,11 +10,12 @@ dependencies {
    compile(project(":kotlin-script-runtime"))
    compile(kotlinStdlib())
    compile(project(":kotlin-scripting-common"))
+    testCompile(commonDep("junit"))
 }

 sourceSets {
    "main" { projectDefault() }
-    "test" {}
+    "test" { projectDefault() }
 }

 publish()
@@ -0,0 +1,146 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.script.experimental.jvm.util
+
+import java.io.File
+import java.io.FileInputStream
+import java.io.IOException
+import java.io.InputStream
+import java.net.JarURLConnection
+import java.net.URL
+import java.util.jar.JarFile
+import java.util.jar.JarInputStream
+import kotlin.script.experimental.jvm.impl.toFileOrNull
+
+fun ClassLoader.forAllMatchingFiles(namePattern: String, body: (String, InputStream) -> Unit) {
+    val processedDirs = HashSet<File>()
+    val processedJars = HashSet<URL>()
+    val nameRegex = namePatternToRegex(namePattern)
+
+    fun iterateResources(vararg keyResourcePaths: String) {
+        for (keyResourcePath in keyResourcePaths) {
+            val resourceRootCalc = ClassLoaderResourceRootFIlePathCalculator(keyResourcePath)
+            for (url in getResources(keyResourcePath)) {
+                if (url.protocol == "jar") {
+                    val jarConnection = url.openConnection() as? JarURLConnection
+                    val jarUrl = jarConnection?.jarFileURL
+                    if (jarUrl != null && !processedJars.contains(jarUrl)) {
+                        processedJars.add(jarUrl)
+                        try {
+                            jarConnection.jarFile
+                        } catch (_: IOException) {
+                            // TODO: consider error reporting
+                            null
+                        }?.let {
+                            forAllMatchingFilesInJarFile(it, nameRegex, body)
+                        }
+                    }
+                } else {
+                    val rootDir = url.toFileOrNull()?.let { resourceRootCalc(it) }
+                    if (rootDir != null && rootDir.isDirectory && !processedDirs.contains(rootDir)) {
+                        processedDirs.add(rootDir)
+                        forAllMatchingFilesInDirectory(rootDir, namePattern, body)
+                    }
+                }
+            }
+        }
+    }
+
+    iterateResources("", JAR_MANIFEST_RESOURCE_NAME)
+}
+
+internal val wildcardChars = "*?".toCharArray()
+internal val patternCharsToEscape = ".*?+()[]^\${}|".toCharArray().also { assert(wildcardChars.all { wc -> it.contains(wc) }) }
+
+private fun Char.escape(): String = (if (patternCharsToEscape.contains(this)) "\\" else "") + this
+
+internal val pathSeparatorChars = "/".let { if (File.separatorChar == '/') it else it + File.separator }.toCharArray()
+internal val pathElementPattern = if (File.separatorChar == '/') "[^/]*" else "[^/${File.separatorChar.escape()}]*"
+internal val pathSeparatorPattern = if (File.separatorChar == '/') "/" else "[/${File.separatorChar.escape()}]."
+internal val specialPatternChars = patternCharsToEscape + pathSeparatorChars
+
+internal fun forAllMatchingFilesInDirectory(baseDir: File, namePattern: String, body: (String, InputStream) -> Unit) {
+    val patternStart = namePattern.indexOfAny(wildcardChars)
+    if (patternStart < 0) {
+        // assuming a single file
+        baseDir.resolve(namePattern).takeIf { it.exists() && it.isFile }?.let { file ->
+            body(file.relativeToOrSelf(baseDir).path, file.inputStream())
+        }
+    } else {
+        val patternDirStart = namePattern.lastIndexOfAny(pathSeparatorChars, patternStart)
+        val root = if (patternDirStart <= 0) baseDir else baseDir.resolve(namePattern.substring(0, patternDirStart))
+        if (root.exists() && root.isDirectory) {
+            val re = namePatternToRegex(namePattern.substring(patternDirStart + 1))
+            root.walkTopDown().filter {
+                re.matches(it.relativeToOrSelf(root).path)
+            }.forEach { file ->
+                body(file.relativeToOrSelf(baseDir).path, file.inputStream())
+            }
+        }
+    }
+}
+
+internal fun forAllMatchingFilesInJarStream(jarInputStream: JarInputStream, nameRegex: Regex, body: (String, InputStream) -> Unit) {
+    do {
+        val entry = jarInputStream.nextJarEntry
+        if (entry != null) {
+            try {
+                if (!entry.isDirectory && nameRegex.matches(entry.name)) {
+                    body(entry.name, jarInputStream)
+                }
+            } finally {
+                jarInputStream.closeEntry()
+            }
+        }
+    } while (entry != null)
+}
+
+internal fun forAllMatchingFilesInJar(jarFile: File, nameRegex: Regex, body: (String, InputStream) -> Unit) {
+    JarInputStream(FileInputStream(jarFile)).use {
+        forAllMatchingFilesInJarStream(it, nameRegex, body)
+    }
+}
+
+internal fun forAllMatchingFilesInJarFile(jarFile: JarFile, nameRegex: Regex, body: (String, InputStream) -> Unit) {
+    jarFile.entries().asSequence().forEach { entry ->
+        if (!entry.isDirectory && nameRegex.matches(entry.name)) {
+            jarFile.getInputStream(entry).use { stream ->
+                body(entry.name, stream)
+            }
+        }
+    }
+}
+
+internal fun namePatternToRegex(pattern: String): Regex = Regex(
+    buildString {
+        var current = 0
+        loop@ while (current < pattern.length) {
+            val nextIndex = pattern.indexOfAny(specialPatternChars, current)
+            val next = if (nextIndex < 0) pattern.length else nextIndex
+            append(pattern.substring(current, next))
+            current = next + 1
+            when {
+                next >= pattern.length -> break@loop
+
+                pathSeparatorChars.contains(pattern[next]) -> append(pathSeparatorPattern)
+
+                pattern[next] == '?' -> append('.')
+
+                pattern[next] == '*' && next + 1 < pattern.length && pattern[next + 1] == '*' -> {
+                    append(".*")
+                    current++
+                }
+
+                pattern[next] == '*' -> append(pathElementPattern)
+
+                else -> {
+                    append('\\')
+                    append(pattern[next])
+                }
+            }
+        }
+    }
+)
@@ -39,7 +39,7 @@ internal const val KOTLIN_COMPILER_JAR = "$KOTLIN_COMPILER_NAME.jar"
 private val JAR_COLLECTIONS_CLASSES_PATHS = arrayOf("BOOT-INF/classes", "WEB-INF/classes")
 private val JAR_COLLECTIONS_LIB_PATHS = arrayOf("BOOT-INF/lib", "WEB-INF/lib")
 private val JAR_COLLECTIONS_KEY_PATHS = JAR_COLLECTIONS_CLASSES_PATHS + JAR_COLLECTIONS_LIB_PATHS
-private const val JAR_MANIFEST_RESOURCE_NAME = "META-INF/MANIFEST.MF"
+internal const val JAR_MANIFEST_RESOURCE_NAME = "META-INF/MANIFEST.MF"

 internal const val KOTLIN_SCRIPT_CLASSPATH_PROPERTY = "kotlin.script.classpath"
 internal const val KOTLIN_COMPILER_CLASSPATH_PROPERTY = "kotlin.compiler.classpath"
@@ -108,20 +108,28 @@ private fun ClassLoader.classPathFromGetUrlsMethodOrNull(): Sequence<File>? {
    }
 }

+internal class ClassLoaderResourceRootFIlePathCalculator(private val keyResourcePath: String) {
+    private var keyResourcePathDepth = -1
+
+    operator fun invoke(resourceFile: File): File {
+        if (keyResourcePathDepth < 0) {
+            keyResourcePathDepth = if (keyResourcePath.isBlank()) 0 else (keyResourcePath.trim('/').count { it == '/' } + 1)
+        }
+        var root = resourceFile
+        for (i in 0 until keyResourcePathDepth) {
+            root = root.parentFile
+        }
+        return root
+    }
+}
+
 internal fun ClassLoader.rawClassPathFromKeyResourcePath(keyResourcePath: String): Sequence<File> {
-    var keyResourcePathDepth = -1
+    val resourceRootCalc = ClassLoaderResourceRootFIlePathCalculator(keyResourcePath)
    return getResources(keyResourcePath).asSequence().mapNotNull { url ->
        if (url.protocol == "jar") {
            (url.openConnection() as? JarURLConnection)?.jarFileURL?.toFileOrNull()
-        } else url.toFileOrNull()?.let { file ->
-            if (keyResourcePathDepth < 0) {
-                keyResourcePathDepth = if (keyResourcePath.isBlank()) 0 else (keyResourcePath.trim('/').count { it == '/' } + 1)
-            }
-            var root = file
-            for (i in 0 until keyResourcePathDepth) {
-                root = root.parentFile
-            }
-            root
+        } else {
+            url.toFileOrNull()?.let { resourceRootCalc(it) }
        }
    }
 }
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.script.experimental.jvm.test
+
+import junit.framework.TestCase
+import org.junit.Test
+import java.io.File
+import java.util.jar.JarFile
+import java.util.jar.JarInputStream
+import kotlin.script.experimental.jvm.util.*
+
+class UtilsTest : TestCase() {
+
+    @Test
+    fun testPatternConversionWildcards() {
+        assertPattern("a${pathSeparatorPattern}b\\.$pathElementPattern", "a/b.*")
+        assertPattern("a$pathSeparatorPattern$pathElementPattern\\.txt", "a/*.txt")
+        assertPattern("a$pathSeparatorPattern.*/b", "a/**/b")
+        assertPattern("a${pathSeparatorPattern}b.\\.txt", "a/b?.txt")
+        assertPattern("$pathElementPattern/b\\.txt", "*/b.txt")
+        assertPattern(".*${pathSeparatorPattern}b\\.txt", "**/b.txt")
+    }
+
+    @Test
+    fun testPatternConversionEscaping() {
+        assertPattern("aa\\+\\(\\)\\[\\]\\^\\\$\\{\\}\\|", "aa+()[]^\${}|")
+        assertPattern("\\+\\(\\)\\[\\]\\^\\\$\\{\\}\\|bb", "+()[]^\${}|bb")
+    }
+
+    @Test
+    fun testSelectFilesInDir() {
+
+        val rootDir = File(".")
+
+        fun assertProjectFilesBy(pattern: String, vararg paths: String) {
+            val res = ArrayList<Pair<String, String>>()
+
+            forAllMatchingFilesInDirectory(rootDir, pattern) { path, stream ->
+                res.add(path to stream.reader().readText())
+            }
+            assertEquals(paths.toSet(), res.mapTo(HashSet()) { it.first })
+
+            res.forEach { (path, bytes) ->
+                val data = File(path).readText()
+                assertEquals("Mismatching data for $path", data, bytes)
+            }
+        }
+
+        assertProjectFilesBy("*.kt") // none
+        assertProjectFilesBy("**/sss/*.kt") // none
+        assertProjectFilesBy(
+            "src/kotlin/script/experimental/jvm/util/jvmClassLoaderUtil.kt",
+            "src/kotlin/script/experimental/jvm/util/jvmClassLoaderUtil.kt"
+        )
+        assertProjectFilesBy(
+            "src/kotlin/script/experimental/jvm/util/jvm?lassLoaderUtil.kt",
+            "src/kotlin/script/experimental/jvm/util/jvmClassLoaderUtil.kt"
+        )
+        assertProjectFilesBy(
+            "src/kotlin/script/experimental/jvm/util/jvm*LoaderUtil.kt",
+            "src/kotlin/script/experimental/jvm/util/jvmClassLoaderUtil.kt"
+        )
+        assertProjectFilesBy("**/jvmClassLoaderUtil.kt", "src/kotlin/script/experimental/jvm/util/jvmClassLoaderUtil.kt")
+        assertProjectFilesBy("**/script/**/jvmClassLoaderUtil.kt", "src/kotlin/script/experimental/jvm/util/jvmClassLoaderUtil.kt")
+        assertProjectFilesBy("src/**/jvmClassLoaderUtil.kt", "src/kotlin/script/experimental/jvm/util/jvmClassLoaderUtil.kt")
+        assertProjectFilesBy("test/**/?????Test.*", "test/kotlin/script/experimental/jvm/test/utilsTest.kt")
+
+        val allSrcKtFiles = HashSet<String>()
+        forAllMatchingFilesInDirectory(rootDir, "src/**/*.kt") { path, _ ->
+            allSrcKtFiles.add(path)
+        }
+        val allExpectedSrcKtFiles =
+            rootDir.walkTopDown().filter {
+                it.relativeToOrSelf(rootDir).path.startsWith("src") && it.extension == "kt"
+            }.mapTo(HashSet()) {
+                it.relativeToOrSelf(rootDir).path
+            }
+        assertEquals(allExpectedSrcKtFiles, allSrcKtFiles)
+    }
+
+    @Test
+    fun testSelectFilesInJar() {
+
+        fun JarFile.filesBy(pattern: String): Map<String, String> {
+            val res = HashMap<String, String>()
+            forAllMatchingFilesInJarFile(this, namePatternToRegex(pattern)) { path, stream ->
+                res[path] = stream.reader().readText().trim()
+            }
+            return res
+        }
+
+        fun JarInputStream.filesBy(pattern: String): Map<String, String> {
+            val res = HashMap<String, String>()
+            forAllMatchingFilesInJarStream(this, namePatternToRegex(pattern)) { path, stream ->
+                res[path] = stream.reader().readText().trim()
+            }
+            return res
+        }
+
+        fun assertFiles(actual: Map<String, String>, vararg expected: Pair<String, String>) {
+            val expectedAsMap = expected.toMap()
+            assertEquals(expectedAsMap, actual)
+        }
+
+        fun assertMatchingFilesInJarTwoWay(jar: File, pattern: String, vararg expected: Pair<String, String>) {
+            assertFiles( JarFile(jar).filesBy(pattern), *expected)
+            assertFiles( JarInputStream(jar.inputStream()).use { it.filesBy(pattern) }, *expected)
+        }
+
+        val jar = File("testData/testJar.jar")
+        assertTrue(jar.exists())
+
+        assertMatchingFilesInJarTwoWay(jar, "META-INF/*.kotlin_module", "META-INF/abc.kotlin_module" to "module")
+        assertMatchingFilesInJarTwoWay(jar, "META-INF/*.kotlin") // none
+        assertMatchingFilesInJarTwoWay(jar, "**/*.class", "a/b/c/d1.class" to "d1", "a/b/c/d1\$s1.class" to "d1s1")
+        assertMatchingFilesInJarTwoWay(jar, "**/*\$*.class", "a/b/c/d1\$s1.class" to "d1s1")
+    }
+
+    private fun assertPattern(expected: String, pattern: String) {
+        assertEquals(expected, namePatternToRegex(pattern).pattern)
+    }
+
+}