azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Kapt: Detect memory leaks in annotation processors (KT-28025)

Browse Source
This commit is contained in:
Yan Zhulanow
2018-11-02 14:53:21 +09:00
parent eb3511164f
commit abd1646d42
9 changed files with 201 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/util/src/org/jetbrains/kotlin/utils/kapt/MemoryLeakDetector.kt
+157
View File
@@ -0,0 +1,157 @@
/*
* Copyright 2010-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
* that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.utils.kapt
import java.lang.ref.WeakReference
import java.lang.reflect.Field
import java.lang.reflect.Modifier
import java.util.*
import javax.annotation.processing.*
import javax.lang.model.AnnotatedConstruct
import javax.lang.model.util.Elements
import javax.lang.model.util.Types
import kotlin.ConcurrentModificationException
class MemoryLeak(val className: String, val fieldName: String, val description: String) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as MemoryLeak
if (className != other.className) return false
if (fieldName != other.fieldName) return false
return true
}
override fun hashCode(): Int {
var result = className.hashCode()
result = 31 * result + fieldName.hashCode()
return result
}
}
private class ClassLoaderData(classLoader: ClassLoader) {
val ref = WeakReference(classLoader)
@Volatile
var age: Int = 0
}
object MemoryLeakDetector {
private val classLoaderData = mutableListOf<ClassLoaderData>()
fun add(classLoader: ClassLoader) {
synchronized(classLoaderData) {
classLoaderData.add(ClassLoaderData(classLoader))
}
}
fun process(): Set<MemoryLeak> {
val memoryLeaks = mutableSetOf<MemoryLeak>()
synchronized(classLoaderData) {
val newClassLoaderData = mutableListOf<ClassLoaderData>()
for (data in classLoaderData) {
val classLoader = data.ref.get() ?: continue
data.age += 1
if (data.age >= 5) {
// Inspect statics just once.
// Note the 'data' is not added to 'nextClassLoaderData' used the next time.
inspectStatics(classLoader)
} else {
newClassLoaderData += data
}
}
classLoaderData.clear()
classLoaderData.addAll(newClassLoaderData)
}
return memoryLeaks
}
private fun inspectStatics(classLoader: ClassLoader): Set<MemoryLeak> {
val loadedClasses = classLoader.loadedClasses()
val loadedClassesSet = try {
loadedClasses.mapTo(mutableSetOf()) { it }
} catch (e: ConcurrentModificationException) {
Thread.sleep(100)
return inspectStatics(classLoader)
}
val leaks = mutableSetOf<MemoryLeak>()
for (clazz in loadedClassesSet) {
val declaredFields = try {
clazz.declaredFields
} catch (e: Throwable) {
continue
}
nextField@ for (field in declaredFields) {
if (!Modifier.isStatic(field.modifiers)) continue
val value = field.getSafe(null)
?.takeIf { !it.isPrimitiveOrString() } ?: continue@nextField
if (value.isJavacComponent()) {
leaks += MemoryLeak(clazz.name, field.name, "Field leaks an Annotation Processing component ($value).")
} else if (value is Class<*> && value in loadedClassesSet) {
leaks += MemoryLeak(clazz.name, field.name, "Field leaks a class type from the same ClassLoader (${value.name}).")
}
}
}
return leaks
}
}
private fun Any.isJavacComponent(): Boolean {
@Suppress("Reformat")
return when (this) {
is Processor, is ProcessingEnvironment, is RoundEnvironment,
is Filer, is Messager, is Elements, is Types, is AnnotatedConstruct -> true
else -> false
}
}
private fun Any.isPrimitiveOrString(): Boolean {
@Suppress("Reformat")
return when (this) {
is Boolean, is Byte, is Short, is Int, is Long,
is Char, is Float, is Double, is Void, is String -> true
else -> false
}
}
private fun ClassLoader.loadedClasses(): Vector<Class<*>> {
try {
val classesField = ClassLoader::class.java.getDeclaredField("classes")
@Suppress("UNCHECKED_CAST")
return classesField.getSafe(this) as? Vector<Class<*>> ?: Vector()
} catch (e: Throwable) {
return Vector()
}
}
private fun Field.getSafe(obj: Any?): Any? {
return try {
val oldIsAccessible = isAccessible
try {
isAccessible = true
get(obj)
} finally {
isAccessible = oldIsAccessible
}
} catch (e: Throwable) {
null
}
}
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/internal/kapt/Kapt3KotlinGradleSubplugin.kt
+1
View File
@@ -343,6 +343,7 @@ class Kapt3KotlinGradleSubplugin : KotlinGradleSubplugin<KotlinCompile> {
pluginOptions += SubpluginOption("correctErrorTypes", "${kaptExtension.correctErrorTypes}")
pluginOptions += SubpluginOption("mapDiagnosticLocations", "${kaptExtension.mapDiagnosticLocations}")
pluginOptions += SubpluginOption("strictMode", "${kaptExtension.strictMode}")
pluginOptions += SubpluginOption("detectMemoryLeaks", "${kaptExtension.detectMemoryLeaks}")
pluginOptions += SubpluginOption("infoAsWarnings", "${project.isInfoAsWarnings()}")
pluginOptions += FilesSubpluginOption("stubs", listOf(getKaptStubsDir()))
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KaptExtension.kt
+2
View File
@@ -34,6 +34,8 @@ open class KaptExtension {
open var strictMode: Boolean = false
open var detectMemoryLeaks: Boolean = true
@Deprecated("Use `annotationProcessor()` and `annotationProcessors()` instead")
open var processors: String = ""
plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/Kapt.kt
+1 -1
View File
@@ -47,7 +47,7 @@ object Kapt {
val processors = processorLoader.loadProcessors(findClassLoaderWithJavac())
val annotationProcessingTime = measureTimeMillis {
kaptContext.doAnnotationProcessing(javaSourceFiles, processors)
kaptContext.doAnnotationProcessing(javaSourceFiles, processors.processors)
}
logger.info { "Annotation processing took $annotationProcessingTime ms" }
plugins/kapt3/kapt3-base/src/org/jetbrains/kotlin/kapt3/base/ProcessorLoader.kt
+4 -2
View File
@@ -14,6 +14,8 @@ import java.net.URLClassLoader
import java.util.*
import javax.annotation.processing.Processor
class LoadedProcessors(val processors: List<Processor>, val classLoader: ClassLoader)
open class ProcessorLoader(
private val paths: KaptPaths,
private val annotationProcessorFqNames: List<String>,
@@ -21,7 +23,7 @@ open class ProcessorLoader(
) : Closeable {
private var annotationProcessingClassLoader: URLClassLoader? = null
fun loadProcessors(parentClassLoader: ClassLoader = ClassLoader.getSystemClassLoader()): List<Processor> {
fun loadProcessors(parentClassLoader: ClassLoader = ClassLoader.getSystemClassLoader()): LoadedProcessors {
clearJarURLCache()
val classpath = (paths.annotationProcessingClasspath + paths.compileClasspath).distinct()
@@ -42,7 +44,7 @@ open class ProcessorLoader(
logger.info { "Annotation processors: " + processors.joinToString { it::class.java.canonicalName } }
}
return processors
return LoadedProcessors(processors, classLoader)
}
open fun doLoadProcessors(classLoader: URLClassLoader): List<Processor> {
plugins/kapt3/kapt3-cli/src/KaptCliOption.kt
+2
View File
@@ -141,6 +141,8 @@ enum class KaptCliOption(
cliToolOption = CliToolOption("-Kapt-strict", FLAG)
),
DETECT_MEMORY_LEAKS_OPTION("detectMemoryLeaks", "true | false", "Detect memory leaks in annotation processors"),
INFO_AS_WARNINGS_OPTION("infoAsWarnings", "true | false", "Show information messages as warnings"),
@Deprecated("Do not use in CLI")
plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Extension.kt
+25 -10
View File
@@ -41,10 +41,7 @@ import org.jetbrains.kotlin.context.ProjectContext
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.kapt3.AptMode.APT_ONLY
import org.jetbrains.kotlin.kapt3.AptMode.WITH_COMPILATION
import org.jetbrains.kotlin.kapt3.base.KaptContext
import org.jetbrains.kotlin.kapt3.base.KaptPaths
import org.jetbrains.kotlin.kapt3.base.ProcessorLoader
import org.jetbrains.kotlin.kapt3.base.doAnnotationProcessing
import org.jetbrains.kotlin.kapt3.base.*
import org.jetbrains.kotlin.kapt3.base.stubs.KaptStubLineInformation.Companion.KAPT_METADATA_EXTENSION
import org.jetbrains.kotlin.kapt3.base.util.KaptBaseError
import org.jetbrains.kotlin.kapt3.base.util.getPackageNameJava9Aware
@@ -58,6 +55,7 @@ import org.jetbrains.kotlin.psi.KtFile
import org.jetbrains.kotlin.resolve.BindingContext
import org.jetbrains.kotlin.resolve.BindingTrace
import org.jetbrains.kotlin.resolve.jvm.extensions.PartialAnalysisHandlerExtension
import org.jetbrains.kotlin.utils.kapt.MemoryLeakDetector
import java.io.File
import java.io.StringWriter
import java.io.Writer
@@ -75,12 +73,13 @@ class ClasspathBasedKapt3Extension(
correctErrorTypes: Boolean,
mapDiagnosticLocations: Boolean,
strictMode: Boolean,
detectMemoryLeaks: Boolean,
pluginInitializedTime: Long,
logger: MessageCollectorBackedKaptLogger,
compilerConfiguration: CompilerConfiguration
) : AbstractKapt3Extension(
paths, options, javacOptions, annotationProcessorFqNames,
aptMode, pluginInitializedTime, logger, correctErrorTypes, mapDiagnosticLocations, strictMode,
aptMode, pluginInitializedTime, logger, correctErrorTypes, mapDiagnosticLocations, strictMode, detectMemoryLeaks,
compilerConfiguration
) {
override val analyzePartially: Boolean
@@ -88,7 +87,7 @@ class ClasspathBasedKapt3Extension(
private var processorLoader: ProcessorLoader? = null
override fun loadProcessors(): List<Processor> {
override fun loadProcessors(): LoadedProcessors {
val efficientProcessorLoader = object : ProcessorLoader(paths, annotationProcessorFqNames, logger) {
override fun doLoadProcessors(classLoader: URLClassLoader): List<Processor> {
return ServiceLoaderLite.loadImplementations(Processor::class.java, classLoader)
@@ -133,6 +132,7 @@ abstract class AbstractKapt3Extension(
val correctErrorTypes: Boolean,
val mapDiagnosticLocations: Boolean,
val strictMode: Boolean,
val detectMemoryLeaks: Boolean,
val compilerConfiguration: CompilerConfiguration
) : PartialAnalysisHandlerExtension() {
private var annotationProcessingComplete = false
@@ -186,7 +186,7 @@ abstract class AbstractKapt3Extension(
if (!aptMode.runAnnotationProcessing) return doNotGenerateCode()
val processors = loadProcessors()
if (processors.isEmpty()) return if (aptMode != WITH_COMPILATION) doNotGenerateCode() else null
if (processors.processors.isEmpty()) return if (aptMode != WITH_COMPILATION) doNotGenerateCode() else null
val kaptContext = KaptContext(paths, false, logger, mapDiagnosticLocations, options, javacOptions)
@@ -230,17 +230,32 @@ abstract class AbstractKapt3Extension(
}
}
private fun runAnnotationProcessing(kaptContext: KaptContext, processors: List<Processor>) {
private fun runAnnotationProcessing(kaptContext: KaptContext, processors: LoadedProcessors) {
if (!aptMode.runAnnotationProcessing) return
val javaSourceFiles = paths.collectJavaSourceFiles()
logger.info { "Java source files: " + javaSourceFiles.joinToString { it.canonicalPath } }
val (annotationProcessingTime) = measureTimeMillis {
kaptContext.doAnnotationProcessing(javaSourceFiles, processors)
kaptContext.doAnnotationProcessing(javaSourceFiles, processors.processors)
}
logger.info { "Annotation processing took $annotationProcessingTime ms" }
if (detectMemoryLeaks) {
MemoryLeakDetector.add(processors.classLoader)
val (leakDetectionTime, leaks) = measureTimeMillis { MemoryLeakDetector.process() }
logger.info { "Leak detection took $leakDetectionTime ms" }
for (leak in leaks) {
logger.warn(buildString {
appendln("Memory leak detected!")
appendln("Location: '${leak.className}', static field '${leak.fieldName}'")
append(leak.description)
})
}
}
}
private fun contextForStubGeneration(
@@ -344,7 +359,7 @@ abstract class AbstractKapt3Extension(
)
}
protected abstract fun loadProcessors(): List<Processor>
protected abstract fun loadProcessors(): LoadedProcessors
}
internal fun JCTree.prettyPrint(context: Context): String {
plugins/kapt3/kapt3-compiler/src/org/jetbrains/kotlin/kapt3/Kapt3Plugin.kt
+6 -1
View File
@@ -115,6 +115,9 @@ object Kapt3ConfigurationKeys {
val STRICT_MODE: CompilerConfigurationKey<String> =
CompilerConfigurationKey.create<String>(STRICT_MODE_OPTION.description)
val DETECT_MEMORY_LEAKS: CompilerConfigurationKey<String> =
CompilerConfigurationKey.create<String>(DETECT_MEMORY_LEAKS_OPTION.description)
}
class Kapt3CommandLineProcessor : CommandLineProcessor {
@@ -147,6 +150,7 @@ class Kapt3CommandLineProcessor : CommandLineProcessor {
MAP_DIAGNOSTIC_LOCATIONS_OPTION -> configuration.put(Kapt3ConfigurationKeys.MAP_DIAGNOSTIC_LOCATIONS, value)
INFO_AS_WARNINGS_OPTION -> configuration.put(Kapt3ConfigurationKeys.INFO_AS_WARNINGS, value)
STRICT_MODE_OPTION -> configuration.put(Kapt3ConfigurationKeys.STRICT_MODE, value)
DETECT_MEMORY_LEAKS_OPTION -> configuration.put(Kapt3ConfigurationKeys.DETECT_MEMORY_LEAKS, value)
CONFIGURATION -> configuration.applyOptionsFrom(decodePluginOptions(value), pluginOptions)
TOOLS_JAR_OPTION -> throw CliOptionProcessingException("'${TOOLS_JAR_OPTION.optionName}' is only supported in the kapt CLI tool")
}
@@ -199,6 +203,7 @@ class Kapt3ComponentRegistrar : ComponentRegistrar {
val isVerbose = configuration.get(Kapt3ConfigurationKeys.VERBOSE_MODE) == "true"
val infoAsWarnings = configuration.get(Kapt3ConfigurationKeys.INFO_AS_WARNINGS) == "true"
val strictMode = configuration.get(Kapt3ConfigurationKeys.STRICT_MODE) == "true"
val detectMemoryLeaks = configuration.get(Kapt3ConfigurationKeys.DETECT_MEMORY_LEAKS) != "false"
val messageCollector = configuration.get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)
?: PrintingMessageCollector(System.err, MessageRenderer.PLAIN_FULL_PATHS, isVerbose)
val logger = MessageCollectorBackedKaptLogger(isVerbose, infoAsWarnings, messageCollector)
@@ -291,7 +296,7 @@ class Kapt3ComponentRegistrar : ComponentRegistrar {
val kapt3AnalysisCompletedHandlerExtension = ClasspathBasedKapt3Extension(
paths, apOptions, javacOptions, annotationProcessors,
aptMode, useLightAnalysis, correctErrorTypes, mapDiagnosticLocations, strictMode,
aptMode, useLightAnalysis, correctErrorTypes, mapDiagnosticLocations, strictMode, detectMemoryLeaks,
System.currentTimeMillis(), logger, configuration
)
plugins/kapt3/kapt3-compiler/test/org/jetbrains/kotlin/kapt3/test/AbstractKotlinKapt3IntegrationTest.kt
+3 -2
View File
@@ -28,6 +28,7 @@ import org.jetbrains.kotlin.kapt3.*
import org.jetbrains.kotlin.kapt3.AptMode.STUBS_AND_APT
import org.jetbrains.kotlin.kapt3.base.KaptContext
import org.jetbrains.kotlin.kapt3.base.KaptPaths
import org.jetbrains.kotlin.kapt3.base.LoadedProcessors
import org.jetbrains.kotlin.kapt3.javac.KaptJavaFileObject
import org.jetbrains.kotlin.kapt3.stubs.ClassFileToSourceStubConverter
import org.jetbrains.kotlin.kapt3.stubs.ClassFileToSourceStubConverter.KaptStub
@@ -174,13 +175,13 @@ abstract class AbstractKotlinKapt3IntegrationTest : CodegenTestCase() {
emptyList(), javaSourceRoots, outputDir, outputDir, stubsOutputDir, incrementalDataOutputDir
), options, emptyMap(), emptyList(), STUBS_AND_APT, System.currentTimeMillis(),
MessageCollectorBackedKaptLogger(true),
correctErrorTypes = true, mapDiagnosticLocations = true, strictMode = true,
correctErrorTypes = true, mapDiagnosticLocations = true, strictMode = true, detectMemoryLeaks = false,
compilerConfiguration = CompilerConfiguration.EMPTY
) {
internal var savedStubs: String? = null
internal var savedBindings: Map<String, KaptJavaFileObject>? = null
override fun loadProcessors() = processors
override fun loadProcessors() = LoadedProcessors(processors, Kapt3ExtensionForTests::class.java.classLoader)
override fun saveStubs(kaptContext: KaptContext, stubs: List<KaptStub>) {
if (this.savedStubs != null) {