Memory leak in PermGen of Kotlin Gradle plugin when demon is used (KT-5420)

Stop background thread in Intellij Idea api and clear thread locals to allow gc the created class loader

 #KT-5420 In Progress

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/com/intellij/openapi/util/LowMemoryWatcher.java

@@ -19,7 +19,9 @@ import com.intellij.openapi.Disposable;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.util.ConcurrencyUtil;
 import com.intellij.util.containers.WeakList;
+import org.jetbrains.annotations.NotNull;
 
+import javax.management.ListenerNotFoundException;
 import javax.management.Notification;
 import javax.management.NotificationEmitter;
 import javax.management.NotificationListener;
@@ -33,9 +35,10 @@ import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
 /**
- * @author Eugene Zhuravlev
+ * PATCHED VERSION of com.intellij.openapi.util.LowMemoryWatcher
- *         Date: Aug 24, 2010
+ * shutdown() method added to stop background thread.
  */
+@SuppressWarnings("UnusedDeclaration")
 public class LowMemoryWatcher {
     private static final long MEM_THRESHOLD = 5 /*MB*/ * 1024 * 1024;
 
@@ -68,6 +71,8 @@ public class LowMemoryWatcher {
 
     private final Runnable myRunnable;
 
+    private static final NotificationListener lowMemoryListener;
+
     static {
         for (MemoryPoolMXBean bean : ManagementFactory.getMemoryPoolMXBeans()) {
             if (bean.getType() == MemoryType.HEAP && bean.isUsageThresholdSupported()) {
@@ -78,10 +83,12 @@ public class LowMemoryWatcher {
                 }
             }
         }
-        ((NotificationEmitter)ManagementFactory.getMemoryMXBean()).addNotificationListener(new NotificationListener() {
+
+        lowMemoryListener = new NotificationListener() {
             @Override
-            public void handleNotification(Notification n, Object hb) {
+            public void handleNotification(@NotNull Notification n, Object hb) {
-                if (MemoryNotificationInfo.MEMORY_THRESHOLD_EXCEEDED.equals(n.getType()) || MemoryNotificationInfo.MEMORY_COLLECTION_THRESHOLD_EXCEEDED.equals(n.getType())) {
+                if (MemoryNotificationInfo.MEMORY_THRESHOLD_EXCEEDED.equals(n.getType()) ||
+                    MemoryNotificationInfo.MEMORY_COLLECTION_THRESHOLD_EXCEEDED.equals(n.getType())) {
                     synchronized (ourJanitor) {
                         if (!ourSubmitted) {
                             //noinspection AssignmentToStaticFieldFromInstanceMethod
@@ -91,7 +98,9 @@ public class LowMemoryWatcher {
                     }
                 }
             }
-        }, null, null);
+        };
+
+        ((NotificationEmitter)ManagementFactory.getMemoryMXBean()).addNotificationListener(lowMemoryListener, null, null);
     }
 
     /**
@@ -126,4 +135,15 @@ public class LowMemoryWatcher {
         ourInstances.remove(this);
     }
 
+    public static void shutdown() throws InterruptedException, ListenerNotFoundException {
+        ourInstances.clear();
+        ((NotificationEmitter)ManagementFactory.getMemoryMXBean()).removeNotificationListener(lowMemoryListener);
+
+        ourExecutor.shutdown();
+
+        boolean terminated = ourExecutor.awaitTermination(1000, TimeUnit.MICROSECONDS);
+        if (!terminated) {
+            ourExecutor.shutdownNow();
+        }
+    }
 }

libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/com/intellij/openapi/util/io/ZipFileCache.java

@@ -27,30 +27,15 @@ import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.zip.ZipFile;
 
 /**
- * <p>Utility class which tries to keep frequently requested .zip files open
+ * PATCHED VERSION of com.intellij.openapi.util.io.ZipFileCache
- * to avoid time loss on closing/reopening ZipFile instances.</p>
+ * shutdown() method added to stop background thread.
- *
- * <p>Clients obtain a file by calling {@link #acquire(String)}
- * and indicate the loss of interest to it via {@link #release(ZipFile)}.
- * Released files are closed after some period of time (about 30 seconds),
- * unless requested again within the period.</p>
- *
- * <p>Since ZipFiles are read-only objects allowing concurrent access,
- * a same instance may be returned to a different threads requesting a same path.
- * A file may be closed only after being released by all applicants.</p>
- *
- * <p>The class does not expect .zip files on a disk to be changed,
- * so it may return an outdated instance of ZipFile (reading from it
- * may return inaccurate data or even cause an exceptions to happen).
- * It's a clients' responsibility to keep a track of .zip files
- * and call the {@link #reset(Collection)} method for a paths
- * which are possibly changed. Reset paths are removed from the cache
- * and are closed immediately after being released.</p>
  */
+@SuppressWarnings("UnusedDeclaration")
 public class ZipFileCache {
     private static final int PERIOD = 10000;   // disposer schedule, ms
     private static final int TIMEOUT = 30000;  // released file close delay, ms
@@ -72,8 +57,12 @@ public class ZipFileCache {
     private static final Map<ZipFile, CacheRecord> ourFileCache = ContainerUtil.newHashMap();
     private static final Map<ZipFile, Integer> ourQueue = ContainerUtil.newHashMap();
 
+    private static final ScheduledThreadPoolExecutor scheduledCloseExecutor;
+
     static {
-        ConcurrencyUtil.newSingleScheduledThreadExecutor("ZipFileCache Dispose", Thread.MIN_PRIORITY).scheduleWithFixedDelay(new Runnable() {
+        scheduledCloseExecutor =
+                ConcurrencyUtil.newSingleScheduledThreadExecutor("ZipFileCache Dispose", Thread.MIN_PRIORITY);
+        scheduledCloseExecutor.scheduleWithFixedDelay(new Runnable() {
             @Override
             public void run() {
                 List<ZipFile> toClose = getFilesToClose(0, System.currentTimeMillis() - TIMEOUT);
@@ -236,4 +225,13 @@ public class ZipFileCache {
     private static void debug(@NotNull String format, Object... args) {
         LogUtil.debug(logger(), format, args);
     }
+
+    public static void shutdown() throws InterruptedException {
+        scheduledCloseExecutor.shutdown();
+
+        boolean terminated = scheduledCloseExecutor.awaitTermination(TIMEOUT, TimeUnit.MICROSECONDS);
+        if (!terminated) {
+            scheduledCloseExecutor.shutdownNow();
+        }
+    }
 }

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/FinishBuildListner.kt

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2010-2014 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.gradle.plugin
+
+import org.gradle.BuildAdapter
+import org.gradle.api.logging.Logging
+import org.gradle.BuildResult
+import java.lang.ref.Reference
+import java.lang.reflect.Array
+import kotlin.platform.platformStatic
+
+class FinishBuildListener(var pluginClassLoader: ParentLastURLClassLoader?) : BuildAdapter() {
+    val log = Logging.getLogger(this.javaClass)
+
+    class object {
+        platformStatic
+        fun isRequestedClass(fqName: String): Boolean {
+            return when (fqName) {
+                "com.intellij.openapi.util.io.ZipFileCache",
+                "com.intellij.openapi.util.LowMemoryWatcher" -> true
+                else -> false
+            }
+        }
+    }
+
+    override fun buildFinished(result: BuildResult?) {
+        log.debug("Build finished listener")
+
+        stopZipFileCache()
+        stopLowMemoryWatcher()
+
+		// TODO: Try to clean up thread locals without this ugly hack
+		// TODO: Further investigation of PermGen leak (KT-6451)
+        removeThreadLocals()
+
+        pluginClassLoader = null
+        result?.getGradle()?.removeListener(this)
+    }
+
+    public fun removeThreadLocals() {
+        try {
+            log.debug("Remove ChildURLClassLoader thread locals")
+
+            val thread = Thread.currentThread()
+            val threadLocalsField = javaClass<Thread>().getDeclaredField("threadLocals")
+            threadLocalsField.setAccessible(true)
+
+            val threadLocalMapClass = Class.forName("java.lang.ThreadLocal\$ThreadLocalMap")
+            val tableField = threadLocalMapClass.getDeclaredField("table")
+            tableField.setAccessible(true)
+
+            val referentField = javaClass<Reference<Any>>().getDeclaredField("referent")
+            referentField.setAccessible(true)
+
+            val table = tableField[threadLocalsField[thread]] as kotlin.Array<*>
+
+            for (entry in table) {
+                if (entry != null) {
+                    val threadLocal = referentField[entry] as ThreadLocal<*>?
+                    val classLoader = threadLocal?.javaClass?.getClassLoader()
+                    if (classLoader is ParentLastURLClassLoader.ChildURLClassLoader) {
+                        threadLocal?.remove()
+                    }
+                }
+            }
+
+            log.debug("Removing ChildURLClassLoader thread locals finished successfully")
+        } catch (e: Throwable) {
+            log.debug("Exception during thread locals remove: " + e)
+        }
+    }
+
+    private fun stopZipFileCache() {
+        callVoidStaticMethod("com.intellij.openapi.util.io.ZipFileCache", "shutdown")
+        log.debug("ZipFileCache finished successfully")
+    }
+
+    private fun stopLowMemoryWatcher() {
+        callVoidStaticMethod("com.intellij.openapi.util.LowMemoryWatcher", "shutdown")
+        log.debug("LowMemoryWatcher finished successfully")
+    }
+
+    private fun callVoidStaticMethod(classFqName: String, methodName: String) {
+        val shortName = classFqName.substring(classFqName.lastIndexOf('.') + 1)
+
+        log.debug("Looking for $shortName class")
+        val lowMemoryWatcherClass = Class.forName(classFqName, false, pluginClassLoader)
+
+        log.debug("Looking for $methodName() method")
+        val shutdownMethod = lowMemoryWatcherClass.getMethod(methodName)
+
+        log.debug("Call $shortName.$methodName()")
+        shutdownMethod.invoke(null)
+    }
+}

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt

@@ -4,21 +4,14 @@ import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.artifacts.dsl.DependencyHandler
 import org.gradle.api.artifacts.ConfigurationContainer
-import org.gradle.api.specs.Spec
-import java.io.File
 import java.net.URL
 import org.gradle.api.logging.Logging
 import java.util.Properties
 import java.io.FileNotFoundException
 import org.gradle.api.initialization.dsl.ScriptHandler
-import java.lang.reflect.Method
+import org.gradle.api.invocation.Gradle
 
 abstract class KotlinBasePluginWrapper: Plugin<Project> {
-
-    class object {
-        val pluginVersionsMap: MutableMap<String, Class<Plugin<Project>>> = hashMapOf()
-    }
-
     val log = Logging.getLogger(this.javaClass)
 
     public override fun apply(project: Project) {
@@ -33,7 +26,10 @@ abstract class KotlinBasePluginWrapper: Plugin<Project> {
         val kotlinPluginVersion = loadKotlinVersionFromResource()
         project.getExtensions().getExtraProperties()?.set("kotlin.gradle.plugin.version", kotlinPluginVersion)
 
-        val cls = pluginVersionsMap.getOrElse("$kotlinPluginVersion:${getPluginClassName()}", { loadPluginInIsolatedClassloader(kotlinPluginVersion, sourceBuildScript) })
+        val pluginClassLoader = createPluginIsolatedClassLoader(kotlinPluginVersion, sourceBuildScript)
+
+        val cls = Class.forName(getPluginClassName(), true, pluginClassLoader)
+        log.debug("Plugin class loaded")
 
         val constructor = cls.getConstructor(javaClass<ScriptHandler>())
         val method = cls.getMethod("apply", javaClass<Project>())
@@ -44,9 +40,12 @@ abstract class KotlinBasePluginWrapper: Plugin<Project> {
 
         method?.invoke(plugin, project)
         log.debug("'apply' method invoked successfully")
+
+        val gradle = project.javaClass.getMethod("getGradle").invoke(project) as Gradle
+        gradle.addBuildListener(FinishBuildListener(pluginClassLoader))
     }
 
-    private fun loadPluginInIsolatedClassloader(projectVersion: String, sourceBuildScript: ScriptHandler): Class<Plugin<Project>> {
+    private fun createPluginIsolatedClassLoader(projectVersion: String, sourceBuildScript: ScriptHandler): ParentLastURLClassLoader {
         val dependencyHandler: DependencyHandler = sourceBuildScript.getDependencies()
         val configurationsContainer: ConfigurationContainer = sourceBuildScript.getConfigurations()
 
@@ -61,9 +60,8 @@ abstract class KotlinBasePluginWrapper: Plugin<Project> {
         log.debug("Load plugin in parent-last URL classloader")
         val kotlinPluginClassloader = ParentLastURLClassLoader(kotlinPluginDependencies, this.javaClass.getClassLoader())
         log.debug("Class loader created")
-        val cls = Class.forName(getPluginClassName(), true, kotlinPluginClassloader) as Class<Plugin<Project>>
+
-        log.debug("Plugin class loaded")
+        return kotlinPluginClassloader
-        return cls
     }
 
     private fun loadKotlinVersionFromResource(): String {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/ParentLastURLClassLoader.java

@@ -1,9 +1,13 @@
 package org.jetbrains.kotlin.gradle.plugin;
 
 
+import org.jetbrains.annotations.NotNull;
+
 import java.net.URL;
 import java.net.URLClassLoader;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * A parent-last classloader that will try the child classloader first and then the parent.
@@ -23,7 +27,7 @@ public class ParentLastURLClassLoader extends ClassLoader {
     }
 
     @Override
-    protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+    protected synchronized Class<?> loadClass(@NotNull String name, boolean resolve) throws ClassNotFoundException {
         try {
             // first we try to find a class inside the child classloader
             return childClassLoader.findClass(name);
@@ -41,8 +45,9 @@ public class ParentLastURLClassLoader extends ClassLoader {
             super(parent);
         }
 
+        @NotNull
         @Override
-        public Class<?> findClass(String name) throws ClassNotFoundException {
+        public Class<?> findClass(@NotNull String name) throws ClassNotFoundException {
             return super.findClass(name);
         }
     }
@@ -51,7 +56,9 @@ public class ParentLastURLClassLoader extends ClassLoader {
      * This class delegates (child then parent) for the findClass method for a URLClassLoader.
      * We need this because findClass is protected in URLClassLoader
      */
-    private static class ChildURLClassLoader extends URLClassLoader {
+    public static class ChildURLClassLoader extends URLClassLoader {
+        private final Map<String, Class<?>> cache = new HashMap<String, Class<?>>();
+
         private FindClassClassLoader realParent;
 
         public ChildURLClassLoader(URL[] urls, FindClassClassLoader realParent) {
@@ -60,10 +67,22 @@ public class ParentLastURLClassLoader extends ClassLoader {
             this.realParent = realParent;
         }
 
+
+        @NotNull
         @Override
-        public Class<?> findClass(String name) throws ClassNotFoundException {
+        public Class<?> findClass(@NotNull String name) throws ClassNotFoundException {
             try {
-                // first try to use the URLClassLoader findClass
+                // Replace with FinishBuildListener.isRequestedClass(name) after rewriting this class on Kotlin
+                if (name.equals("com.intellij.openapi.util.io.ZipFileCache") ||
+                        name.equals("com.intellij.openapi.util.LowMemoryWatcher")) {
+                    if (cache.containsKey(name)) return cache.get(name);
+
+                    Class<?> aClass = super.findClass(name);
+                    cache.put(name, aClass);
+
+                    return aClass;
+                }
+
                 return super.findClass(name);
             } catch (ClassNotFoundException e) {
                 // if that fails, we ask our real parent classloader to load the class (we give up)