azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Dropping custom classloading again, now without creating a transparent one, restoring cleanup for gradle <2.4 to minimize leaks under daemon, adding diagnostics

Browse Source 
(cherry picked from commit a3b7be4)
This commit is contained in:
ligee
2015-07-29 13:19:02 +02:00
parent 3b322520bd
commit 0a9f98545b
7 changed files with 78 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/misc.xml
Generated
+1 -1
View File
@@ -54,7 +54,7 @@
<component name="ProjectResources">
<default-html-doctype>http://www.w3.org/1999/xhtml</default-html-doctype>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_6" assert-keyword="true" jdk-15="true" project-jdk-name="1.6" project-jdk-type="JavaSDK">
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_6" default="false" assert-keyword="true" jdk-15="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
<component name="WebServicesPlugin" addRequiredLibraries="true" />
compiler/compiler.pro
+1 -1
View File
@@ -93,7 +93,7 @@ messages/**)
# for gradle plugin and other server tools
-keep class com.intellij.openapi.util.io.ZipFileCache { public *; }
-keep class com.intellij.openapi.util.LowMemoryWatcher { public *; }
#-keep class com.intellij.openapi.util.LowMemoryWatcher { public *; }
# for j2k
-keep class com.intellij.codeInsight.NullableNotNullManager { public protected *; }
libraries/tools/kotlin-gradle-plugin-core/pom.xml
+1 -1
View File
@@ -44,7 +44,7 @@
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-compiler</artifactId>
<artifactId>kotlin-compiler-embeddable</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/com/intellij/openapi/util/LowMemoryWatcher.java
-154
View File
@@ -1,154 +0,0 @@
/*
* Copyright 2000-2013 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.intellij.openapi.util;
import com.intellij.openapi.Disposable;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.util.ConcurrencyUtil;
import com.intellij.util.containers.WeakList;
import org.jetbrains.annotations.NotNull;
import javax.management.ListenerNotFoundException;
import javax.management.Notification;
import javax.management.NotificationEmitter;
import javax.management.NotificationListener;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryNotificationInfo;
import java.lang.management.MemoryPoolMXBean;
import java.lang.management.MemoryType;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* PATCHED VERSION of com.intellij.openapi.util.LowMemoryWatcher
* shutdown() method added to stop background thread.
*/
@SuppressWarnings("UnusedDeclaration")
public class LowMemoryWatcher {
private static final long MEM_THRESHOLD = 5 /*MB*/ * 1024 * 1024;
private static final Logger LOG = Logger.getInstance("#com.intellij.openapi.util.LowMemoryWatcher");
private static final List<LowMemoryWatcher> ourInstances = new WeakList<LowMemoryWatcher>();
private static final ThreadPoolExecutor ourExecutor = new ThreadPoolExecutor(0, 1, 10, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(2), ConcurrencyUtil.newNamedThreadFactory("LowMemoryWatcher janitor"));
private static boolean ourSubmitted;
private static final Runnable ourJanitor = new Runnable() {
@Override
public void run() {
try {
for (LowMemoryWatcher watcher : ourInstances) {
try {
watcher.myRunnable.run();
}
catch (Throwable e) {
LOG.info(e);
}
}
}
finally {
synchronized (ourJanitor) {
//noinspection AssignmentToStaticFieldFromInstanceMethod
ourSubmitted = false;
}
}
}
};
private final Runnable myRunnable;
private static final NotificationListener lowMemoryListener;
static {
for (MemoryPoolMXBean bean : ManagementFactory.getMemoryPoolMXBeans()) {
if (bean.getType() == MemoryType.HEAP && bean.isUsageThresholdSupported()) {
long threshold = bean.getUsage().getMax() - MEM_THRESHOLD;
if (threshold > 0) {
bean.setUsageThreshold(threshold);
bean.setCollectionUsageThreshold(threshold);
}
}
}
lowMemoryListener = new NotificationListener() {
@Override
public void handleNotification(@NotNull Notification n, Object hb) {
if (MemoryNotificationInfo.MEMORY_THRESHOLD_EXCEEDED.equals(n.getType()) ||
MemoryNotificationInfo.MEMORY_COLLECTION_THRESHOLD_EXCEEDED.equals(n.getType())) {
synchronized (ourJanitor) {
if (!ourSubmitted) {
//noinspection AssignmentToStaticFieldFromInstanceMethod
ourSubmitted = true;
ourExecutor.submit(ourJanitor);
}
}
}
}
};
((NotificationEmitter)ManagementFactory.getMemoryMXBean()).addNotificationListener(lowMemoryListener, null, null);
}
/**
* Registers a runnable to run on low memory events
* @return a LowMemoryWatcher instance holding the runnable. This instance should be kept in memory while the
* low memory notification functionality is needed. As soon as it's garbage-collected, the runnable won't receive any further notifications.
*/
public static LowMemoryWatcher register(Runnable runnable) {
return new LowMemoryWatcher(runnable);
}
/**
* Registers a runnable to run on low memory events. The notifications will be issued until parentDisposable is disposed.
*/
public static void register(Runnable runnable, Disposable parentDisposable) {
final Ref<LowMemoryWatcher> watcher = Ref.create(new LowMemoryWatcher(runnable));
Disposer.register(parentDisposable, new Disposable() {
@Override
public void dispose() {
watcher.get().stop();
watcher.set(null);
}
});
}
private LowMemoryWatcher(Runnable runnable) {
myRunnable = runnable;
ourInstances.add(this);
}
public void stop() {
ourInstances.remove(this);
}
/**
* LowMemoryWatcher maintains a background thread where all the handlers are invoked.
* In server environments, this thread may run indefinitely and prevent the class loader from
* being gc-ed. Thus it's necessary to invoke this method to stop that thread and let the classes be garbage-collected.
*/
public static void stopAll() throws InterruptedException, ListenerNotFoundException {
ourInstances.clear();
((NotificationEmitter)ManagementFactory.getMemoryMXBean()).removeNotificationListener(lowMemoryListener);
ourExecutor.shutdown();
boolean terminated = ourExecutor.awaitTermination(50, TimeUnit.MICROSECONDS);
if (!terminated) {
ourExecutor.shutdownNow();
}
}
}
libraries/tools/kotlin-gradle-plugin/pom.xml
-1
View File
@@ -41,7 +41,6 @@
<groupId>org.jetbrains.kotlin</groupId>
<artifactId>kotlin-gradle-plugin-core</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jetbrains.kotlin</groupId>
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/FinishBuildListener.kt
+68 -43
View File
@@ -18,64 +18,89 @@ package org.jetbrains.kotlin.gradle.plugin
import org.gradle.BuildAdapter
import org.gradle.BuildResult
import org.gradle.api.invocation.Gradle
import org.gradle.api.logging.Logging
import java.lang.ref.Reference
import java.util.concurrent.ScheduledExecutorService
import kotlin.text.MatchGroup
class FinishBuildListener(var pluginClassLoader: ParentLastURLClassLoader?) : BuildAdapter() {
internal fun getUsedMemoryKb(): Long {
System.gc()
val rt = Runtime.getRuntime()
return (rt.totalMemory() - rt.freeMemory()) / 1024
}
private fun comparableVersionStr(version: String) =
"(\\d+)\\.(\\d+).*"
.toRegex()
.match(version)
?.groups
?.drop(1)?.take(2)
// checking if two subexpression groups are found and length of each is >0 and <4
?.let { if (it.all { (it?.value?.length() ?: 0).let { it > 0 && it < 4 }}) it else null }
?.map { "%3s".format(it?.value ?: 0).replace(' ', '0') }
?.joinToString(".")
class FinishBuildListener(pluginClassLoader: ClassLoader?, val startMemory: Long) : BuildAdapter() {
val log = Logging.getLogger(this.javaClass)
private var threadTracker: ThreadTracker? = ThreadTracker()
private val cleanup = CompilerServicesCleanup(pluginClassLoader)
override fun buildFinished(result: BuildResult?) {
log.kotlinDebug("Build finished listener")
stopZipFileCache()
stopLowMemoryWatcher()
stopJobScheduler()
// TODO: Try to clean up thread locals without this ugly hack
// TODO: Further investigation of PermGen leak (KT-6451)
removeThreadLocals()
pluginClassLoader = null
result?.getGradle()?.removeListener(this)
threadTracker?.checkThreadLeak(result?.getGradle())
threadTracker = null
}
public fun removeThreadLocals() {
try {
log.kotlinDebug("Remove ChildURLClassLoader thread locals")
val thread = Thread.currentThread()
val threadLocalsField = javaClass<Thread>().getDeclaredField("threadLocals")
threadLocalsField.setAccessible(true)
val threadLocalMapClass = Class.forName("java.lang.ThreadLocal\$ThreadLocalMap")
val tableField = threadLocalMapClass.getDeclaredField("table")
tableField.setAccessible(true)
val referentField = javaClass<Reference<Any>>().getDeclaredField("referent")
referentField.setAccessible(true)
val table = tableField[threadLocalsField[thread]] as Array<*>
for (entry in table) {
if (entry != null) {
val threadLocal = referentField[entry] as ThreadLocal<*>?
val classLoader = threadLocal?.javaClass?.getClassLoader()
if (classLoader is ParentLastURLClassLoader.ChildURLClassLoader) {
threadLocal?.remove()
}
val gradle = result?.getGradle()
if (gradle != null) {
// making cleanup only on recognized versions of gradle and if version < 2.4
// otherwise it may cause problems e.g. with JobScheduler on subsequent runs
// this strategy may lead to memory leaks, but prevent crashes due to destroyed JobScheduler
// the reason for the strategy is the following:
// gradle < 2.4 has problems with plugin reuse in the daemon: new calls to the plugin are made with a new classloader
// for every new build. With statically initialized daemons like JobScheduler that leads to big leaks of classloaders and classes,
// therefore to reduce leaks JobScheduler (and deprecated ZipFileCache for now) should be stopped)
// It should be noted that because of this behavior there are no benefits of using daemon in these versions.
// Starting from 2.4 gradle using cached classloaders, that leads to effective class reusing in the daemon, but
// in that case premature stopping of the static daemons may lead to crashes.
comparableVersionStr(gradle.getGradleVersion())?.let {
log.kotlinDebug("detected gradle version $it")
if (it < comparableVersionStr("2.4")!!) {
cleanup()
// checking thread leaks only then cleaning up
threadTracker?.checkThreadLeak(gradle)
}
}
log.kotlinDebug("Removing ChildURLClassLoader thread locals finished successfully")
} catch (e: Throwable) {
log.kotlinDebug("Exception during thread locals remove: " + e)
threadTracker = null
gradle.removeListener(this)
}
// the value reported here is not necessarily a leak, since it is calculated before collecting the plugin classes
// but on subsequent runs in the daemon it should be rather small, then the classes are actually reused by the daemon (see above)
getUsedMemoryKb().let { log.kotlinDebug("[PERF] Used memory after build: $it kb (${"%+d".format(it-startMemory)} kb)") }
}
}
class CompilerServicesCleanup(private var pluginClassLoader: ClassLoader?) {
val log = Logging.getLogger(this.javaClass)
fun invoke() {
assert(pluginClassLoader != null)
log.kotlinDebug("compiler services cleanup")
// \todo remove ZipFileCache cleanup after switching to recent idea libs
stopZipFileCache()
// currently unused since LowMemoryWatcher usages are (hopefully) all removed
// \todo doublecheck and remove LowMemoryWatcher cleanup code
//stopLowMemoryWatcher()
stopJobScheduler()
pluginClassLoader = null
}
private fun stopZipFileCache() {
libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt
+7 -43
View File
@@ -15,15 +15,16 @@ import java.lang.reflect.Method
import org.jetbrains.kotlin.gradle.tasks.KotlinTasksProvider
import java.net.URLClassLoader
// \todo simplify: the complicated structure is a leftover from dynamic loading of plugin core, could be significantly simplified now
abstract class KotlinBasePluginWrapper: Plugin<Project> {
val log = Logging.getLogger(this.javaClass)
companion object {
var pluginClassLoader: ClassLoader? = null
}
public override fun apply(project: Project) {
val startMemory = getUsedMemoryKb()
val sourceBuildScript = findSourceBuildScript(project)
if (sourceBuildScript == null) {
log.error("Failed to determine source cofiguration of kotlin plugin. Can not download core. Please verify that this or any parent project " +
@@ -34,50 +35,14 @@ abstract class KotlinBasePluginWrapper: Plugin<Project> {
val kotlinPluginVersion = loadKotlinVersionFromResource(log)
project.getExtensions().getExtraProperties()?.set("kotlin.gradle.plugin.version", kotlinPluginVersion)
if (pluginClassLoader == null)
pluginClassLoader = createPluginIsolatedClassLoader(kotlinPluginVersion, sourceBuildScript)
else
log.kotlinDebug("Reusing classloader from previous run")
val plugin = getPlugin(pluginClassLoader!!, sourceBuildScript)
val plugin = getPlugin(this.javaClass.getClassLoader(), sourceBuildScript)
plugin.apply(project)
// project.getGradle().addBuildListener(FinishBuildListener(pluginClassLoader))
project.getGradle().addBuildListener(FinishBuildListener(this.javaClass.getClassLoader(), startMemory))
}
protected abstract fun getPlugin(pluginClassLoader: ClassLoader, scriptHandler: ScriptHandler): Plugin<Project>
private fun createPluginClassLoader(projectVersion: String, sourceBuildScript: ScriptHandler): URLClassLoader {
val kotlinPluginDependencies: List<URL> = getPluginDependencies(projectVersion, sourceBuildScript)
log.kotlinDebug("Load plugin in regular URL classloader")
val kotlinPluginClassloader = URLClassLoader(kotlinPluginDependencies.toTypedArray(), this.javaClass.getClassLoader())
return kotlinPluginClassloader
}
private fun createPluginIsolatedClassLoader(projectVersion: String, sourceBuildScript: ScriptHandler): ParentLastURLClassLoader {
val kotlinPluginDependencies: List<URL> = getPluginDependencies(projectVersion, sourceBuildScript)
log.kotlinDebug("Load plugin in parent-last URL classloader")
val kotlinPluginClassloader = ParentLastURLClassLoader(kotlinPluginDependencies, this.javaClass.getClassLoader())
log.kotlinDebug("Class loader created")
return kotlinPluginClassloader
}
private fun getPluginDependencies(projectVersion: String, sourceBuildScript: ScriptHandler): List<URL> {
val dependencyHandler: DependencyHandler = sourceBuildScript.getDependencies()
val configurationsContainer: ConfigurationContainer = sourceBuildScript.getConfigurations()
log.kotlinDebug("Creating configuration and dependency")
val kotlinPluginCoreCoordinates = "org.jetbrains.kotlin:kotlin-gradle-plugin-core:" + projectVersion
val dependency = dependencyHandler.create(kotlinPluginCoreCoordinates)
val configuration = configurationsContainer.detachedConfiguration(dependency)
log.kotlinDebug("Resolving [" + kotlinPluginCoreCoordinates + "]")
val kotlinPluginDependencies: List<URL> = configuration.getResolvedConfiguration().getFiles({ true })!!.map { it.toURI().toURL() }
log.kotlinDebug("Resolved files: [" + kotlinPluginDependencies.toString() + "]")
return kotlinPluginDependencies
}
private fun findSourceBuildScript(project: Project): ScriptHandler? {
log.kotlinDebug("Looking for proper script handler")
var curProject = project
@@ -111,4 +76,3 @@ open class Kotlin2JsPluginWrapper : KotlinBasePluginWrapper() {
fun Logger.kotlinDebug(message: String) {
this.debug("[KOTLIN] $message")
}