From abd1646d42b6c3f0b5be702556357521adf6abd0 Mon Sep 17 00:00:00 2001 From: Yan Zhulanow Date: Fri, 2 Nov 2018 14:53:21 +0900 Subject: [PATCH] Kapt: Detect memory leaks in annotation processors (KT-28025) --- .../kotlin/utils/kapt/MemoryLeakDetector.kt | 157 ++++++++++++++++++ .../kapt/Kapt3KotlinGradleSubplugin.kt | 1 + .../kotlin/gradle/plugin/KaptExtension.kt | 2 + .../org/jetbrains/kotlin/kapt3/base/Kapt.kt | 2 +- .../kotlin/kapt3/base/ProcessorLoader.kt | 6 +- plugins/kapt3/kapt3-cli/src/KaptCliOption.kt | 2 + .../jetbrains/kotlin/kapt3/Kapt3Extension.kt | 35 ++-- .../org/jetbrains/kotlin/kapt3/Kapt3Plugin.kt | 7 +- .../AbstractKotlinKapt3IntegrationTest.kt | 5 +- 9 files changed, 201 insertions(+), 16 deletions(-) create mode 100644 compiler/util/src/org/jetbrains/kotlin/utils/kapt/MemoryLeakDetector.kt diff --git a/compiler/util/src/org/jetbrains/kotlin/utils/kapt/MemoryLeakDetector.kt b/compiler/util/src/org/jetbrains/kotlin/utils/kapt/MemoryLeakDetector.kt new file mode 100644 index 00000000000..e7437f331b4 --- /dev/null +++ b/compiler/util/src/org/jetbrains/kotlin/utils/kapt/MemoryLeakDetector.kt @@ -0,0 +1,157 @@ +/* + * Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license + * that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.utils.kapt + +import java.lang.ref.WeakReference +import java.lang.reflect.Field +import java.lang.reflect.Modifier +import java.util.* +import javax.annotation.processing.* +import javax.lang.model.AnnotatedConstruct +import javax.lang.model.util.Elements +import javax.lang.model.util.Types +import kotlin.ConcurrentModificationException + +class MemoryLeak(val className: String, val fieldName: String, val description: String) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as MemoryLeak + + if (className != other.className) return false + if (fieldName != other.fieldName) return false + + return true + } + + override fun hashCode(): Int { + var result = className.hashCode() + result = 31 * result + fieldName.hashCode() + return result + } +} + +private class ClassLoaderData(classLoader: ClassLoader) { + val ref = WeakReference(classLoader) + + @Volatile + var age: Int = 0 +} + +object MemoryLeakDetector { + private val classLoaderData = mutableListOf() + + fun add(classLoader: ClassLoader) { + synchronized(classLoaderData) { + classLoaderData.add(ClassLoaderData(classLoader)) + } + } + + fun process(): Set { + val memoryLeaks = mutableSetOf() + + synchronized(classLoaderData) { + val newClassLoaderData = mutableListOf() + for (data in classLoaderData) { + val classLoader = data.ref.get() ?: continue + data.age += 1 + + if (data.age >= 5) { + // Inspect statics just once. + // Note the 'data' is not added to 'nextClassLoaderData' used the next time. + inspectStatics(classLoader) + } else { + newClassLoaderData += data + } + } + + classLoaderData.clear() + classLoaderData.addAll(newClassLoaderData) + } + + return memoryLeaks + } + + private fun inspectStatics(classLoader: ClassLoader): Set { + val loadedClasses = classLoader.loadedClasses() + val loadedClassesSet = try { + loadedClasses.mapTo(mutableSetOf()) { it } + } catch (e: ConcurrentModificationException) { + Thread.sleep(100) + return inspectStatics(classLoader) + } + + val leaks = mutableSetOf() + + for (clazz in loadedClassesSet) { + val declaredFields = try { + clazz.declaredFields + } catch (e: Throwable) { + continue + } + + nextField@ for (field in declaredFields) { + if (!Modifier.isStatic(field.modifiers)) continue + + val value = field.getSafe(null) + ?.takeIf { !it.isPrimitiveOrString() } ?: continue@nextField + + if (value.isJavacComponent()) { + leaks += MemoryLeak(clazz.name, field.name, "Field leaks an Annotation Processing component ($value).") + } else if (value is Class<*> && value in loadedClassesSet) { + leaks += MemoryLeak(clazz.name, field.name, "Field leaks a class type from the same ClassLoader (${value.name}).") + } + } + } + + return leaks + } +} + +private fun Any.isJavacComponent(): Boolean { + @Suppress("Reformat") + return when (this) { + is Processor, is ProcessingEnvironment, is RoundEnvironment, + is Filer, is Messager, is Elements, is Types, is AnnotatedConstruct -> true + else -> false + } +} + +private fun Any.isPrimitiveOrString(): Boolean { + @Suppress("Reformat") + return when (this) { + is Boolean, is Byte, is Short, is Int, is Long, + is Char, is Float, is Double, is Void, is String -> true + else -> false + } +} + +private fun ClassLoader.loadedClasses(): Vector> { + try { + val classesField = ClassLoader::class.java.getDeclaredField("classes") + + @Suppress("UNCHECKED_CAST") + return classesField.getSafe(this) as? Vector> ?: Vector() + } catch (e: Throwable) { + return Vector() + } +} + +private fun Field.getSafe(obj: Any?): Any? { + return try { + val oldIsAccessible = isAccessible + + try { + isAccessible = true + get(obj) + } finally { + isAccessible = oldIsAccessible + } + } catch (e: Throwable) { + null + } +} \ No newline at end of file diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/Kapt3KotlinGradleSubplugin.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/Kapt3KotlinGradleSubplugin.kt index 193ba8eb70a..651e8e4f790 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/Kapt3KotlinGradleSubplugin.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/Kapt3KotlinGradleSubplugin.kt @@ -343,6 +343,7 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin { pluginOptions += SubpluginOption("correctErrorTypes", "${kaptExtension.correctErrorTypes}") pluginOptions += SubpluginOption("mapDiagnosticLocations", "${kaptExtension.mapDiagnosticLocations}") pluginOptions += SubpluginOption("strictMode", "${kaptExtension.strictMode}") + pluginOptions += SubpluginOption("detectMemoryLeaks", "${kaptExtension.detectMemoryLeaks}") pluginOptions += SubpluginOption("infoAsWarnings", "${project.isInfoAsWarnings()}") pluginOptions += FilesSubpluginOption("stubs", listOf(getKaptStubsDir())) diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KaptExtension.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KaptExtension.kt index fad9e713b51..f7ea5a4790e 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KaptExtension.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KaptExtension.kt @@ -34,6 +34,8 @@ open class KaptExtension { open var strictMode: Boolean = false + open var detectMemoryLeaks: Boolean = true + @Deprecated("Use `annotationProcessor()` and `annotationProcessors()` instead") open var processors: String = "" diff --git a/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/Kapt.kt b/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/Kapt.kt index 8144f2ad7fc..d64dd1f734a 100644 --- a/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/Kapt.kt +++ b/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/Kapt.kt @@ -47,7 +47,7 @@ object Kapt { val processors = processorLoader.loadProcessors(findClassLoaderWithJavac()) val annotationProcessingTime = measureTimeMillis { - kaptContext.doAnnotationProcessing(javaSourceFiles, processors) + kaptContext.doAnnotationProcessing(javaSourceFiles, processors.processors) } logger.info { "Annotation processing took $annotationProcessingTime ms" } diff --git a/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/ProcessorLoader.kt b/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/ProcessorLoader.kt index 497f43f7cd2..6969dd686db 100644 --- a/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/ProcessorLoader.kt +++ b/plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/ProcessorLoader.kt @@ -14,6 +14,8 @@ import java.net.URLClassLoader import java.util.* import javax.annotation.processing.Processor +class LoadedProcessors(val processors: List, val classLoader: ClassLoader) + open class ProcessorLoader( private val paths: KaptPaths, private val annotationProcessorFqNames: List, @@ -21,7 +23,7 @@ open class ProcessorLoader( ) : Closeable { private var annotationProcessingClassLoader: URLClassLoader? = null - fun loadProcessors(parentClassLoader: ClassLoader = ClassLoader.getSystemClassLoader()): List { + fun loadProcessors(parentClassLoader: ClassLoader = ClassLoader.getSystemClassLoader()): LoadedProcessors { clearJarURLCache() val classpath = (paths.annotationProcessingClasspath + paths.compileClasspath).distinct() @@ -42,7 +44,7 @@ open class ProcessorLoader( logger.info { "Annotation processors: " + processors.joinToString { it::class.java.canonicalName } } } - return processors + return LoadedProcessors(processors, classLoader) } open fun doLoadProcessors(classLoader: URLClassLoader): List { diff --git a/plugins/kapt3/kapt3-cli/src/KaptCliOption.kt b/plugins/kapt3/kapt3-cli/src/KaptCliOption.kt index 7a14ebc7ff8..3037f4de7d2 100644 --- a/plugins/kapt3/kapt3-cli/src/KaptCliOption.kt +++ b/plugins/kapt3/kapt3-cli/src/KaptCliOption.kt @@ -141,6 +141,8 @@ enum class KaptCliOption( cliToolOption = CliToolOption("-Kapt-strict", FLAG) ), + DETECT_MEMORY_LEAKS_OPTION("detectMemoryLeaks", "true | false", "Detect memory leaks in annotation processors"), + INFO_AS_WARNINGS_OPTION("infoAsWarnings", "true | false", "Show information messages as warnings"), @Deprecated("Do not use in CLI") diff --git a/plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Extension.kt b/plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Extension.kt index 6b5fd5ad53e..fd3a7cc3113 100644 --- a/plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Extension.kt +++ b/plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Extension.kt @@ -41,10 +41,7 @@ import org.jetbrains.kotlin.context.ProjectContext import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.kapt3.AptMode.APT_ONLY import org.jetbrains.kotlin.kapt3.AptMode.WITH_COMPILATION -import org.jetbrains.kotlin.kapt3.base.KaptContext -import org.jetbrains.kotlin.kapt3.base.KaptPaths -import org.jetbrains.kotlin.kapt3.base.ProcessorLoader -import org.jetbrains.kotlin.kapt3.base.doAnnotationProcessing +import org.jetbrains.kotlin.kapt3.base.* import org.jetbrains.kotlin.kapt3.base.stubs.KaptStubLineInformation.Companion.KAPT_METADATA_EXTENSION import org.jetbrains.kotlin.kapt3.base.util.KaptBaseError import org.jetbrains.kotlin.kapt3.base.util.getPackageNameJava9Aware @@ -58,6 +55,7 @@ import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.resolve.BindingContext import org.jetbrains.kotlin.resolve.BindingTrace import org.jetbrains.kotlin.resolve.jvm.extensions.PartialAnalysisHandlerExtension +import org.jetbrains.kotlin.utils.kapt.MemoryLeakDetector import java.io.File import java.io.StringWriter import java.io.Writer @@ -75,12 +73,13 @@ class ClasspathBasedKapt3Extension( correctErrorTypes: Boolean, mapDiagnosticLocations: Boolean, strictMode: Boolean, + detectMemoryLeaks: Boolean, pluginInitializedTime: Long, logger: MessageCollectorBackedKaptLogger, compilerConfiguration: CompilerConfiguration ) : AbstractKapt3Extension( paths, options, javacOptions, annotationProcessorFqNames, - aptMode, pluginInitializedTime, logger, correctErrorTypes, mapDiagnosticLocations, strictMode, + aptMode, pluginInitializedTime, logger, correctErrorTypes, mapDiagnosticLocations, strictMode, detectMemoryLeaks, compilerConfiguration ) { override val analyzePartially: Boolean @@ -88,7 +87,7 @@ class ClasspathBasedKapt3Extension( private var processorLoader: ProcessorLoader? = null - override fun loadProcessors(): List { + override fun loadProcessors(): LoadedProcessors { val efficientProcessorLoader = object : ProcessorLoader(paths, annotationProcessorFqNames, logger) { override fun doLoadProcessors(classLoader: URLClassLoader): List { return ServiceLoaderLite.loadImplementations(Processor::class.java, classLoader) @@ -133,6 +132,7 @@ abstract class AbstractKapt3Extension( val correctErrorTypes: Boolean, val mapDiagnosticLocations: Boolean, val strictMode: Boolean, + val detectMemoryLeaks: Boolean, val compilerConfiguration: CompilerConfiguration ) : PartialAnalysisHandlerExtension() { private var annotationProcessingComplete = false @@ -186,7 +186,7 @@ abstract class AbstractKapt3Extension( if (!aptMode.runAnnotationProcessing) return doNotGenerateCode() val processors = loadProcessors() - if (processors.isEmpty()) return if (aptMode != WITH_COMPILATION) doNotGenerateCode() else null + if (processors.processors.isEmpty()) return if (aptMode != WITH_COMPILATION) doNotGenerateCode() else null val kaptContext = KaptContext(paths, false, logger, mapDiagnosticLocations, options, javacOptions) @@ -230,17 +230,32 @@ abstract class AbstractKapt3Extension( } } - private fun runAnnotationProcessing(kaptContext: KaptContext, processors: List) { + private fun runAnnotationProcessing(kaptContext: KaptContext, processors: LoadedProcessors) { if (!aptMode.runAnnotationProcessing) return val javaSourceFiles = paths.collectJavaSourceFiles() logger.info { "Java source files: " + javaSourceFiles.joinToString { it.canonicalPath } } val (annotationProcessingTime) = measureTimeMillis { - kaptContext.doAnnotationProcessing(javaSourceFiles, processors) + kaptContext.doAnnotationProcessing(javaSourceFiles, processors.processors) } logger.info { "Annotation processing took $annotationProcessingTime ms" } + + if (detectMemoryLeaks) { + MemoryLeakDetector.add(processors.classLoader) + + val (leakDetectionTime, leaks) = measureTimeMillis { MemoryLeakDetector.process() } + logger.info { "Leak detection took $leakDetectionTime ms" } + + for (leak in leaks) { + logger.warn(buildString { + appendln("Memory leak detected!") + appendln("Location: '${leak.className}', static field '${leak.fieldName}'") + append(leak.description) + }) + } + } } private fun contextForStubGeneration( @@ -344,7 +359,7 @@ abstract class AbstractKapt3Extension( ) } - protected abstract fun loadProcessors(): List + protected abstract fun loadProcessors(): LoadedProcessors } internal fun JCTree.prettyPrint(context: Context): String { diff --git a/plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Plugin.kt b/plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Plugin.kt index b456ef9a33f..82e2119bde6 100644 --- a/plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Plugin.kt +++ b/plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Plugin.kt @@ -115,6 +115,9 @@ object Kapt3ConfigurationKeys { val STRICT_MODE: CompilerConfigurationKey = CompilerConfigurationKey.create(STRICT_MODE_OPTION.description) + + val DETECT_MEMORY_LEAKS: CompilerConfigurationKey = + CompilerConfigurationKey.create(DETECT_MEMORY_LEAKS_OPTION.description) } class Kapt3CommandLineProcessor : CommandLineProcessor { @@ -147,6 +150,7 @@ class Kapt3CommandLineProcessor : CommandLineProcessor { MAP_DIAGNOSTIC_LOCATIONS_OPTION -> configuration.put(Kapt3ConfigurationKeys.MAP_DIAGNOSTIC_LOCATIONS, value) INFO_AS_WARNINGS_OPTION -> configuration.put(Kapt3ConfigurationKeys.INFO_AS_WARNINGS, value) STRICT_MODE_OPTION -> configuration.put(Kapt3ConfigurationKeys.STRICT_MODE, value) + DETECT_MEMORY_LEAKS_OPTION -> configuration.put(Kapt3ConfigurationKeys.DETECT_MEMORY_LEAKS, value) CONFIGURATION -> configuration.applyOptionsFrom(decodePluginOptions(value), pluginOptions) TOOLS_JAR_OPTION -> throw CliOptionProcessingException("'${TOOLS_JAR_OPTION.optionName}' is only supported in the kapt CLI tool") } @@ -199,6 +203,7 @@ class Kapt3ComponentRegistrar : ComponentRegistrar { val isVerbose = configuration.get(Kapt3ConfigurationKeys.VERBOSE_MODE) == "true" val infoAsWarnings = configuration.get(Kapt3ConfigurationKeys.INFO_AS_WARNINGS) == "true" val strictMode = configuration.get(Kapt3ConfigurationKeys.STRICT_MODE) == "true" + val detectMemoryLeaks = configuration.get(Kapt3ConfigurationKeys.DETECT_MEMORY_LEAKS) != "false" val messageCollector = configuration.get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY) ?: PrintingMessageCollector(System.err, MessageRenderer.PLAIN_FULL_PATHS, isVerbose) val logger = MessageCollectorBackedKaptLogger(isVerbose, infoAsWarnings, messageCollector) @@ -291,7 +296,7 @@ class Kapt3ComponentRegistrar : ComponentRegistrar { val kapt3AnalysisCompletedHandlerExtension = ClasspathBasedKapt3Extension( paths, apOptions, javacOptions, annotationProcessors, - aptMode, useLightAnalysis, correctErrorTypes, mapDiagnosticLocations, strictMode, + aptMode, useLightAnalysis, correctErrorTypes, mapDiagnosticLocations, strictMode, detectMemoryLeaks, System.currentTimeMillis(), logger, configuration ) diff --git a/plugins/kapt3/kapt3-compiler/test/org/jetbrains/kotlin/kapt3/test/AbstractKotlinKapt3IntegrationTest.kt b/plugins/kapt3/kapt3-compiler/test/org/jetbrains/kotlin/kapt3/test/AbstractKotlinKapt3IntegrationTest.kt index 8dafa65e576..0025c1cc5c3 100644 --- a/plugins/kapt3/kapt3-compiler/test/org/jetbrains/kotlin/kapt3/test/AbstractKotlinKapt3IntegrationTest.kt +++ b/plugins/kapt3/kapt3-compiler/test/org/jetbrains/kotlin/kapt3/test/AbstractKotlinKapt3IntegrationTest.kt @@ -28,6 +28,7 @@ import org.jetbrains.kotlin.kapt3.* import org.jetbrains.kotlin.kapt3.AptMode.STUBS_AND_APT import org.jetbrains.kotlin.kapt3.base.KaptContext import org.jetbrains.kotlin.kapt3.base.KaptPaths +import org.jetbrains.kotlin.kapt3.base.LoadedProcessors import org.jetbrains.kotlin.kapt3.javac.KaptJavaFileObject import org.jetbrains.kotlin.kapt3.stubs.ClassFileToSourceStubConverter import org.jetbrains.kotlin.kapt3.stubs.ClassFileToSourceStubConverter.KaptStub @@ -174,13 +175,13 @@ abstract class AbstractKotlinKapt3IntegrationTest : CodegenTestCase() { emptyList(), javaSourceRoots, outputDir, outputDir, stubsOutputDir, incrementalDataOutputDir ), options, emptyMap(), emptyList(), STUBS_AND_APT, System.currentTimeMillis(), MessageCollectorBackedKaptLogger(true), - correctErrorTypes = true, mapDiagnosticLocations = true, strictMode = true, + correctErrorTypes = true, mapDiagnosticLocations = true, strictMode = true, detectMemoryLeaks = false, compilerConfiguration = CompilerConfiguration.EMPTY ) { internal var savedStubs: String? = null internal var savedBindings: Map? = null - override fun loadProcessors() = processors + override fun loadProcessors() = LoadedProcessors(processors, Kapt3ExtensionForTests::class.java.classLoader) override fun saveStubs(kaptContext: KaptContext, stubs: List) { if (this.savedStubs != null) {