Switch plugin and annotation processor loading to the own implementation of ServiceLoader

'ServiceLoader' in JDK8 leaks file handles (https://bugs.openjdk.java.net/browse/JDK-8156014). New implementation uses the ZipFile API, it also doesn't operate on the whole classpath which is not often needed.
2018-10-29 16:12:15 +09:00
parent 11e23ecc70
commit 0ffa901859
8 changed files with 459 additions and 6 deletions
@@ -1,14 +1,20 @@
-
 import java.io.File
 import org.gradle.api.tasks.bundling.Jar
 import org.jetbrains.kotlin.gradle.dsl.KotlinCompile
+import org.jetbrains.kotlin.gradle.dsl.KotlinJvmCompile

 plugins {
    kotlin("jvm")
    id("jps-compatible")
 }

-jvmTarget = "1.6"
+tasks.named<KotlinJvmCompile>("compileKotlin") {
+    kotlinOptions.jvmTarget = "1.6"
+}
+
+tasks.named<KotlinJvmCompile>("compileTestKotlin") {
+    kotlinOptions.jvmTarget = "1.8"
+}

 val compilerModules: Array<String> by rootProject.extra
 val otherCompilerModules = compilerModules.filter { it != path }
@@ -67,7 +67,7 @@ object PluginCliParser {
            this::class.java.classLoader
        )

-        val componentRegistrars = ServiceLoader.load(ComponentRegistrar::class.java, classLoader).toMutableList()
+        val componentRegistrars = ServiceLoaderLite.loadImplementations(ComponentRegistrar::class.java, classLoader)
        configuration.addAll(ComponentRegistrar.PLUGIN_COMPONENT_REGISTRARS, componentRegistrars)

        processPluginOptions(pluginOptions, configuration, classLoader)
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.cli.jvm.plugins
+
+import java.io.File
+import java.io.IOError
+import java.lang.Character.isJavaIdentifierPart
+import java.lang.Character.isJavaIdentifierStart
+import java.lang.IllegalArgumentException
+import java.lang.RuntimeException
+import java.net.URLClassLoader
+import java.util.zip.ZipFile
+
+/**
+ * ServiceLoader has a file handle leak in JDK8: https://bugs.openjdk.java.net/browse/JDK-8156014.
+ * This class, hopefully, doesn't. :)
+ */
+object ServiceLoaderLite {
+    private const val SERVICE_DIRECTORY_LOCATION = "META-INF/services/"
+
+    class ServiceLoadingException(val file: File, cause: Throwable) : RuntimeException("Error loading services from $file", cause)
+
+    /**
+     * Returns implementations for the given `service` declared in META-INF/services of the `classLoader` roots.
+     *
+     * Note that the behavior is radically different from what Java ServiceLoader does.
+     * ServiceLoaderLite doesn't iterate over the whole ClassLoader hierarchy, it takes only the immediate roots of `classLoader`.
+     * In fact, this is often the desired behavior.
+     */
+    fun <Service> loadImplementations(service: Class<out Service>, classLoader: URLClassLoader): List<Service> {
+        val files = classLoader.urLs.map { url ->
+            if (url.protocol.toLowerCase() != "file") throw IllegalArgumentException("Only local files are supported, got $url")
+            val path = url.path.takeIf { it.isNotEmpty() } ?: throw IllegalArgumentException("Path is empty for $url")
+            File(path)
+        }
+
+        val implementations = mutableListOf<Service>()
+
+        for (className in findImplementations(service, files)) {
+            val instance = Class.forName(className, false, classLoader).newInstance()
+            implementations += service.cast(instance)
+        }
+
+        return implementations
+    }
+
+    inline fun <reified Service : Any> findImplementations(files: List<File>): Set<String> {
+        return findImplementations(Service::class.java, files)
+    }
+
+    inline fun <reified Service : Any> loadImplementations(classLoader: URLClassLoader): List<Service> {
+        return loadImplementations(Service::class.java, classLoader)
+    }
+
+    fun findImplementations(service: Class<*>, files: List<File>): Set<String> {
+        return files.flatMapTo(linkedSetOf()) { findImplementations(service, it) }
+    }
+
+    private fun findImplementations(service: Class<*>, file: File): Set<String> {
+        val classIdentifier = getClassIdentifier(service)
+
+        return when {
+            file.isDirectory -> findImplementationsInDirectory(classIdentifier, file)
+            file.isFile -> findImplementationsInJar(classIdentifier, file)
+            else -> emptySet()
+        }
+    }
+
+    private fun findImplementationsInDirectory(classId: String, file: File): Set<String> {
+        val serviceFile = File(file, SERVICE_DIRECTORY_LOCATION + classId).takeIf { it.isFile } ?: return emptySet()
+
+        try {
+            return serviceFile.useLines { parseLines(file, it) }
+        } catch (e: IOError) {
+            throw ServiceLoadingException(file, e)
+        }
+    }
+
+    private fun findImplementationsInJar(classId: String, file: File): Set<String> {
+        ZipFile(file).use { zipFile ->
+            val entry = zipFile.getEntry(SERVICE_DIRECTORY_LOCATION + classId) ?: return emptySet()
+            zipFile.getInputStream(entry).use { inputStream ->
+                return inputStream.bufferedReader().useLines { parseLines(file, it) }
+            }
+        }
+    }
+
+    private fun parseLines(file: File, lines: Sequence<String>): Set<String> {
+        return lines.mapNotNullTo(linkedSetOf()) { parseLine(file, it) }
+    }
+
+    private fun parseLine(file: File, line: String): String? {
+        val actualLine = line.substringBefore('#').trim().takeIf { it.isNotEmpty() } ?: return null
+
+        actualLine.forEachIndexed { index: Int, c: Char ->
+            val isValid = if (index == 0) isJavaIdentifierStart(c) else isJavaIdentifierPart(c) || c == '.'
+            if (!isValid) {
+                val errorText = "Invalid Java identifier: $line"
+                throw ServiceLoadingException(file, RuntimeException(errorText))
+            }
+        }
+
+        return actualLine
+    }
+
+    private fun getClassIdentifier(service: Class<*>): String {
+        return service.name
+    }
+}
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.cli.serviceLoaderLite
+
+import org.jetbrains.kotlin.test.TestCaseWithTmpdir
+import java.io.File
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
+import javax.annotation.processing.Processor
+
+abstract class AbstractServiceLoaderLiteTest : TestCaseWithTmpdir() {
+    protected fun applyForDirAndJar(name: String, vararg entries: Entry, block: (File) -> Unit) {
+        val zip = writeJar("$name.jar", *entries)
+        block(zip)
+
+        val dir = writeDir(name, *entries)
+        block(dir)
+    }
+
+    protected fun writeDir(dirName: String, vararg entries: Entry): File {
+        val dir = File(tmpdir, dirName)
+        if (dir.exists()) {
+            throw IllegalStateException("Directory $dirName already exists")
+        }
+
+        dir.mkdir()
+
+        for ((name, content) in entries) {
+            val file = File(dir, name).also { it.parentFile.mkdirs() }
+            file.writeBytes(content)
+        }
+
+        return dir
+    }
+
+    protected fun writeJar(fileName: String, vararg entries: Entry): File {
+        val file = File(tmpdir, fileName)
+        if (file.exists()) {
+            throw IllegalStateException("File $fileName already exists")
+        }
+
+        file.outputStream().use { os ->
+            ZipOutputStream(os).use { zos ->
+                for ((name, content) in entries) {
+                    zos.putNextEntry(ZipEntry(name))
+                    zos.write(content)
+                }
+            }
+        }
+
+        return file
+    }
+
+    protected inline fun <reified E : Throwable> assertThrows(block: () -> Unit) {
+        try {
+            block()
+        } catch (e: Throwable) {
+            if (e !is E) {
+                fail(E::class.java.name + " exception expected, got " + e.javaClass.name)
+            }
+            return
+        }
+
+        fail(E::class.java.name + " exception expected, got nothing")
+    }
+
+    protected data class Entry(val name: String, val content: ByteArray) {
+        constructor(name: String, content: String) : this(name, content.toByteArray())
+    }
+
+    protected fun processors(content: String) = Entry(
+        "META-INF/services/" + Processor::class.java.name,
+        content
+    )
+}
@@ -0,0 +1,116 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.cli.serviceLoaderLite
+
+import org.jetbrains.kotlin.cli.jvm.plugins.ServiceLoaderLite
+import org.jetbrains.kotlin.cli.jvm.plugins.ServiceLoaderLite.ServiceLoadingException
+import javax.annotation.processing.Processor
+
+class ServiceLoaderLiteTest : AbstractServiceLoaderLiteTest() {
+    fun testSimple() = applyForDirAndJar("test", processors("test.Foo")) { file ->
+        val impls = ServiceLoaderLite.findImplementations<Processor>(listOf(file))
+        assertEquals("test.Foo", impls.single())
+    }
+
+    fun testEmpty() = applyForDirAndJar("test") { file ->
+        val impls = ServiceLoaderLite.findImplementations<Processor>(listOf(file))
+        assertEquals(0, impls.size)
+    }
+
+    fun testEmpty2() = applyForDirAndJar("test", Entry("foo", "bar")) { file ->
+        val impls = ServiceLoaderLite.findImplementations<Processor>(listOf(file))
+        assertEquals(0, impls.size)
+    }
+
+    fun testEmpty3() = applyForDirAndJar("test", processors("")) { file ->
+        val impls = ServiceLoaderLite.findImplementations<Processor>(listOf(file))
+        assertEquals(0, impls.size)
+    }
+
+    fun testSeveralProcessors() {
+        val processorsContent = buildString { appendln("test.Foo").appendln("test.Bar") }
+
+        applyForDirAndJar("test", processors(processorsContent)) { file ->
+            val impls = ServiceLoaderLite.findImplementations(Processor::class.java, listOf(file))
+            assertEquals(2, impls.size)
+            assertTrue("test.Foo" in impls)
+            assertTrue("test.Bar" in impls)
+        }
+    }
+
+    fun testSeveralEntries() = applyForDirAndJar("test", processors("test.Foo"), Entry("foo", "bar")) { file ->
+        val impls = ServiceLoaderLite.findImplementations<Processor>(listOf(file))
+        assertEquals("test.Foo", impls.single())
+    }
+
+    fun testSeveralJars() {
+        val jar1 = writeJar("test.jar", processors("test.Foo"))
+        val jar2 = writeJar("test2.jar", processors("ap.Bar"))
+
+        val impls = ServiceLoaderLite.findImplementations<Processor>(listOf(jar1, jar2))
+
+        assertEquals(2, impls.size)
+        assertTrue("test.Foo" in impls)
+        assertTrue("ap.Bar" in impls)
+    }
+
+    fun testSeveralDirs() {
+        val dir1 = writeDir("test", processors("test.Foo"))
+        val dir2 = writeDir("test2", processors("ap.Bar"))
+
+        val impls = ServiceLoaderLite.findImplementations<Processor>(listOf(dir1, dir2))
+
+        assertEquals(2, impls.size)
+        assertTrue("test.Foo" in impls)
+        assertTrue("ap.Bar" in impls)
+    }
+
+    fun testDirAndJar() {
+        val jar = writeJar("test", processors("test.Foo"))
+        val dir = writeDir("test2", processors("ap.Bar"))
+
+        val impls = ServiceLoaderLite.findImplementations<Processor>(listOf(jar, dir))
+
+        assertEquals(2, impls.size)
+        assertTrue("test.Foo" in impls)
+        assertTrue("ap.Bar" in impls)
+    }
+
+    fun testParsingError() {
+        applyForDirAndJar("test", processors("5")) { file ->
+            assertThrows<ServiceLoadingException> {
+                ServiceLoaderLite.findImplementations(Processor::class.java, listOf(file))
+            }
+        }
+    }
+
+    fun testParsingError2() {
+        applyForDirAndJar("test", processors("a b c")) { file ->
+            assertThrows<ServiceLoadingException> {
+                ServiceLoaderLite.findImplementations(Processor::class.java, listOf(file))
+            }
+        }
+    }
+
+    fun testCommentsAndWhitespaces() {
+        val processorsContent = buildString {
+            appendln("  test.Foo #comment")
+            appendln("#comment2")
+            appendln().appendln()
+            appendln("test.Bar #anotherComemnt")
+            appendln("test.Zoo  ")
+        }
+
+        applyForDirAndJar("test", processors(processorsContent)) { file ->
+            val impls = ServiceLoaderLite.findImplementations(Processor::class.java, listOf(file))
+            assertEquals(3, impls.size)
+            assertTrue("test.Foo" in impls)
+            assertTrue("test.Bar" in impls)
+            assertTrue("test.Zoo" in impls)
+        }
+    }
+}
+
@@ -0,0 +1,128 @@
+/*
+ * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.cli.serviceLoaderLite
+
+import org.jetbrains.kotlin.cli.jvm.plugins.ServiceLoaderLite
+import java.net.URLClassLoader
+import kotlin.reflect.KClass
+
+interface Intf
+class Component1 : Intf
+class Component2 : Intf
+class ComponentWithParameters(val a: String) : Intf
+class UnrelatedComponent
+enum class EnumComponent : Intf
+
+class ServiceLoaderLiteTestWithClassLoader : AbstractServiceLoaderLiteTest() {
+    class NestedComponent : Intf
+    inner class InnerComponent : Intf
+
+    fun testClassloader1() {
+        @Suppress("RemoveExplicitTypeArguments")
+        val entries = arrayOf(impls<Intf>(Component1::class, Component2::class), clazz<Component1>(), clazz<Component2>())
+
+        classLoaderTest("test", *entries) { classLoader ->
+            val impls = ServiceLoaderLite.loadImplementations<Intf>(classLoader)
+            assertTrue(impls.any { it is Component1 })
+            assertTrue(impls.any { it is Component2 })
+        }
+    }
+
+    fun testNestedComponent() {
+        classLoaderTest("test", impls<Intf>(NestedComponent::class), clazz<NestedComponent>()) { classLoader ->
+            val impls = ServiceLoaderLite.loadImplementations<Intf>(classLoader)
+            assertTrue(impls.single() is NestedComponent)
+        }
+    }
+
+    fun testInnerComponent() {
+        classLoaderTest("test", impls<Intf>(InnerComponent::class), clazz<InnerComponent>()) { classLoader ->
+            assertThrows<InstantiationException> {
+                ServiceLoaderLite.loadImplementations<Intf>(classLoader)
+            }
+        }
+    }
+
+    fun testComponentWithParameters() {
+        classLoaderTest("test", impls<Intf>(ComponentWithParameters::class), clazz<ComponentWithParameters>()) { classLoader ->
+            assertThrows<InstantiationException> {
+                ServiceLoaderLite.loadImplementations<Intf>(classLoader)
+            }
+        }
+    }
+
+    fun testInterface() {
+        @Suppress("RemoveExplicitTypeArguments")
+        classLoaderTest("test", impls<Intf>(Intf::class), clazz<Intf>()) { classLoader ->
+            assertThrows<InstantiationException> {
+                ServiceLoaderLite.loadImplementations<Intf>(classLoader)
+            }
+        }
+    }
+
+    fun testEnum() {
+        classLoaderTest("test", impls<Intf>(EnumComponent::class), clazz<EnumComponent>()) { classLoader ->
+            assertThrows<InstantiationException> {
+                ServiceLoaderLite.loadImplementations<Intf>(classLoader)
+            }
+        }
+    }
+
+    fun testUnrelatedComponent() {
+        val implsEntry = Entry("META-INF/services/" + Intf::class.java.name, UnrelatedComponent::class.java.name)
+        classLoaderTest("test", implsEntry, clazz<UnrelatedComponent>()) { classLoader ->
+            assertThrows<ClassCastException> {
+                ServiceLoaderLite.loadImplementations<Intf>(classLoader)
+            }
+        }
+    }
+
+    fun testNestedClassLoaders() {
+        val entries1 = arrayOf(impls<Intf>(Component1::class), clazz<Component1>())
+        val entries2 = arrayOf(impls<Intf>(Component2::class), clazz<Component2>())
+
+        var index = 0
+        classLoaderTest("test" + index++, *entries1) { classLoader1 ->
+            val impls1 = ServiceLoaderLite.loadImplementations<Intf>(classLoader1)
+            assertTrue(impls1.single() is Component1)
+
+            classLoaderTest("test2" + index++, *entries2, parent = classLoader1) { classLoader2 ->
+                val impls2 = ServiceLoaderLite.loadImplementations<Intf>(classLoader2)
+                assertTrue(impls2.single() is Component2)
+            }
+        }
+    }
+
+    fun testEmpty() {
+        val classLoader = URLClassLoader(emptyArray(), ServiceLoaderLiteTestWithClassLoader::class.java.classLoader)
+        val impls = ServiceLoaderLite.loadImplementations<Intf>(classLoader)
+        assertTrue(impls.isEmpty())
+    }
+
+    private fun classLoaderTest(name: String, vararg entries: Entry, parent: ClassLoader? = null, block: (URLClassLoader) -> Unit) {
+        applyForDirAndJar(name, *entries) { file ->
+            val parentClassLoader = parent ?: ServiceLoaderLiteTestWithClassLoader::class.java.classLoader
+            val classLoader = URLClassLoader(arrayOf(file.toURI().toURL()), parentClassLoader)
+            block(classLoader)
+        }
+    }
+
+    private inline fun <reified T : Any> clazz() = Entry(T::class.java.name.replace('.', '/'), bytecode(T::class.java))
+
+    private fun bytecode(clazz: Class<*>): ByteArray {
+        val resourcePath = clazz.name.replace('.', '/') + ".class"
+        return clazz.classLoader.getResource(resourcePath).readBytes()
+    }
+
+    private inline fun <reified Intf : Any> impls(vararg impls: KClass<out Intf>): Entry {
+        val content = buildString {
+            for (impl in impls) {
+                appendln(impl.java.name)
+            }
+        }
+        return Entry("META-INF/services/" + Intf::class.java.name, content)
+    }
+}
@@ -14,7 +14,7 @@ import java.net.URLClassLoader
 import java.util.*
 import javax.annotation.processing.Processor

-class ProcessorLoader(
+open class ProcessorLoader(
    private val paths: KaptPaths,
    private val annotationProcessorFqNames: List<String>,
    private val logger: KaptLogger
@@ -33,7 +33,7 @@ class ProcessorLoader(
            annotationProcessorFqNames.mapNotNull { tryLoadProcessor(it, classLoader) }
        } else {
            logger.info("Need to discovery annotation processors in the AP classpath")
-            ServiceLoader.load(Processor::class.java, classLoader).toList()
+            doLoadProcessors(classLoader)
        }

        if (processors.isEmpty()) {
@@ -45,6 +45,10 @@ class ProcessorLoader(
        return processors
    }

+    open fun doLoadProcessors(classLoader: URLClassLoader): List<Processor> {
+        return ServiceLoader.load(Processor::class.java, classLoader).toList()
+    }
+
    private fun tryLoadProcessor(fqName: String, classLoader: ClassLoader): Processor? {
        val annotationProcessorClass = try {
            Class.forName(fqName, true, classLoader)
@@ -28,6 +28,7 @@ import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.OUTPUT
 import org.jetbrains.kotlin.cli.common.messages.MessageCollector
 import org.jetbrains.kotlin.cli.common.messages.OutputMessageUtil
 import org.jetbrains.kotlin.cli.common.output.writeAll
+import org.jetbrains.kotlin.cli.jvm.plugins.ServiceLoaderLite
 import org.jetbrains.kotlin.codegen.ClassBuilderMode
 import org.jetbrains.kotlin.codegen.CompilationErrorHandler
 import org.jetbrains.kotlin.codegen.KotlinCodegenFacade
@@ -60,6 +61,7 @@ import org.jetbrains.kotlin.resolve.jvm.extensions.PartialAnalysisHandlerExtensi
 import java.io.File
 import java.io.StringWriter
 import java.io.Writer
+import java.net.URLClassLoader
 import javax.annotation.processing.Processor
 import com.sun.tools.javac.util.List as JavacList

@@ -87,7 +89,14 @@ class ClasspathBasedKapt3Extension(
    private var processorLoader: ProcessorLoader? = null

    override fun loadProcessors(): List<Processor> {
-        return ProcessorLoader(paths, annotationProcessorFqNames, logger).also { this.processorLoader = it }.loadProcessors()
+        val efficientProcessorLoader = object : ProcessorLoader(paths, annotationProcessorFqNames, logger) {
+            override fun doLoadProcessors(classLoader: URLClassLoader): List<Processor> {
+                return ServiceLoaderLite.loadImplementations(Processor::class.java, classLoader)
+            }
+        }
+
+        this.processorLoader = efficientProcessorLoader
+        return efficientProcessorLoader.loadProcessors()
    }

    override fun analysisCompleted(