diff --git a/.idea/misc.xml b/.idea/misc.xml index 4aa68e48699..c7404941a41 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -54,7 +54,7 @@ http://www.w3.org/1999/xhtml - + diff --git a/compiler/compiler.pro b/compiler/compiler.pro index 323072c06c4..1f11a834d47 100644 --- a/compiler/compiler.pro +++ b/compiler/compiler.pro @@ -93,7 +93,7 @@ messages/**) # for gradle plugin and other server tools -keep class com.intellij.openapi.util.io.ZipFileCache { public *; } --keep class com.intellij.openapi.util.LowMemoryWatcher { public *; } +#-keep class com.intellij.openapi.util.LowMemoryWatcher { public *; } # for j2k -keep class com.intellij.codeInsight.NullableNotNullManager { public protected *; } diff --git a/libraries/tools/kotlin-gradle-plugin-core/pom.xml b/libraries/tools/kotlin-gradle-plugin-core/pom.xml index 8636d5d7c2a..62c431995ac 100644 --- a/libraries/tools/kotlin-gradle-plugin-core/pom.xml +++ b/libraries/tools/kotlin-gradle-plugin-core/pom.xml @@ -44,7 +44,7 @@ org.jetbrains.kotlin - kotlin-compiler + kotlin-compiler-embeddable ${project.version} diff --git a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/com/intellij/openapi/util/LowMemoryWatcher.java b/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/com/intellij/openapi/util/LowMemoryWatcher.java deleted file mode 100644 index 654b563f796..00000000000 --- a/libraries/tools/kotlin-gradle-plugin-core/src/main/kotlin/com/intellij/openapi/util/LowMemoryWatcher.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2000-2013 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.intellij.openapi.util; - -import com.intellij.openapi.Disposable; -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.util.ConcurrencyUtil; -import com.intellij.util.containers.WeakList; -import org.jetbrains.annotations.NotNull; - -import javax.management.ListenerNotFoundException; -import javax.management.Notification; -import javax.management.NotificationEmitter; -import javax.management.NotificationListener; -import java.lang.management.ManagementFactory; -import java.lang.management.MemoryNotificationInfo; -import java.lang.management.MemoryPoolMXBean; -import java.lang.management.MemoryType; -import java.util.List; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -/** - * PATCHED VERSION of com.intellij.openapi.util.LowMemoryWatcher - * shutdown() method added to stop background thread. - */ -@SuppressWarnings("UnusedDeclaration") -public class LowMemoryWatcher { - private static final long MEM_THRESHOLD = 5 /*MB*/ * 1024 * 1024; - - private static final Logger LOG = Logger.getInstance("#com.intellij.openapi.util.LowMemoryWatcher"); - - private static final List ourInstances = new WeakList(); - private static final ThreadPoolExecutor ourExecutor = new ThreadPoolExecutor(0, 1, 10, TimeUnit.SECONDS, new ArrayBlockingQueue(2), ConcurrencyUtil.newNamedThreadFactory("LowMemoryWatcher janitor")); - private static boolean ourSubmitted; - private static final Runnable ourJanitor = new Runnable() { - @Override - public void run() { - try { - for (LowMemoryWatcher watcher : ourInstances) { - try { - watcher.myRunnable.run(); - } - catch (Throwable e) { - LOG.info(e); - } - } - } - finally { - synchronized (ourJanitor) { - //noinspection AssignmentToStaticFieldFromInstanceMethod - ourSubmitted = false; - } - } - } - }; - - private final Runnable myRunnable; - - private static final NotificationListener lowMemoryListener; - - static { - for (MemoryPoolMXBean bean : ManagementFactory.getMemoryPoolMXBeans()) { - if (bean.getType() == MemoryType.HEAP && bean.isUsageThresholdSupported()) { - long threshold = bean.getUsage().getMax() - MEM_THRESHOLD; - if (threshold > 0) { - bean.setUsageThreshold(threshold); - bean.setCollectionUsageThreshold(threshold); - } - } - } - - lowMemoryListener = new NotificationListener() { - @Override - public void handleNotification(@NotNull Notification n, Object hb) { - if (MemoryNotificationInfo.MEMORY_THRESHOLD_EXCEEDED.equals(n.getType()) || - MemoryNotificationInfo.MEMORY_COLLECTION_THRESHOLD_EXCEEDED.equals(n.getType())) { - synchronized (ourJanitor) { - if (!ourSubmitted) { - //noinspection AssignmentToStaticFieldFromInstanceMethod - ourSubmitted = true; - ourExecutor.submit(ourJanitor); - } - } - } - } - }; - - ((NotificationEmitter)ManagementFactory.getMemoryMXBean()).addNotificationListener(lowMemoryListener, null, null); - } - - /** - * Registers a runnable to run on low memory events - * @return a LowMemoryWatcher instance holding the runnable. This instance should be kept in memory while the - * low memory notification functionality is needed. As soon as it's garbage-collected, the runnable won't receive any further notifications. - */ - public static LowMemoryWatcher register(Runnable runnable) { - return new LowMemoryWatcher(runnable); - } - - /** - * Registers a runnable to run on low memory events. The notifications will be issued until parentDisposable is disposed. - */ - public static void register(Runnable runnable, Disposable parentDisposable) { - final Ref watcher = Ref.create(new LowMemoryWatcher(runnable)); - Disposer.register(parentDisposable, new Disposable() { - @Override - public void dispose() { - watcher.get().stop(); - watcher.set(null); - } - }); - } - - private LowMemoryWatcher(Runnable runnable) { - myRunnable = runnable; - ourInstances.add(this); - } - - public void stop() { - ourInstances.remove(this); - } - - /** - * LowMemoryWatcher maintains a background thread where all the handlers are invoked. - * In server environments, this thread may run indefinitely and prevent the class loader from - * being gc-ed. Thus it's necessary to invoke this method to stop that thread and let the classes be garbage-collected. - */ - public static void stopAll() throws InterruptedException, ListenerNotFoundException { - ourInstances.clear(); - ((NotificationEmitter)ManagementFactory.getMemoryMXBean()).removeNotificationListener(lowMemoryListener); - - ourExecutor.shutdown(); - - boolean terminated = ourExecutor.awaitTermination(50, TimeUnit.MICROSECONDS); - if (!terminated) { - ourExecutor.shutdownNow(); - } - } -} diff --git a/libraries/tools/kotlin-gradle-plugin/pom.xml b/libraries/tools/kotlin-gradle-plugin/pom.xml index 9252c989dd8..038e68f14fe 100644 --- a/libraries/tools/kotlin-gradle-plugin/pom.xml +++ b/libraries/tools/kotlin-gradle-plugin/pom.xml @@ -41,7 +41,6 @@ org.jetbrains.kotlin kotlin-gradle-plugin-core ${project.version} - test org.jetbrains.kotlin diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/FinishBuildListener.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/FinishBuildListener.kt index f6039acb826..671bd6557d7 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/FinishBuildListener.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/FinishBuildListener.kt @@ -18,64 +18,89 @@ package org.jetbrains.kotlin.gradle.plugin import org.gradle.BuildAdapter import org.gradle.BuildResult +import org.gradle.api.invocation.Gradle import org.gradle.api.logging.Logging import java.lang.ref.Reference import java.util.concurrent.ScheduledExecutorService +import kotlin.text.MatchGroup -class FinishBuildListener(var pluginClassLoader: ParentLastURLClassLoader?) : BuildAdapter() { +internal fun getUsedMemoryKb(): Long { + System.gc() + val rt = Runtime.getRuntime() + return (rt.totalMemory() - rt.freeMemory()) / 1024 +} + + +private fun comparableVersionStr(version: String) = + "(\\d+)\\.(\\d+).*" + .toRegex() + .match(version) + ?.groups + ?.drop(1)?.take(2) + // checking if two subexpression groups are found and length of each is >0 and <4 + ?.let { if (it.all { (it?.value?.length() ?: 0).let { it > 0 && it < 4 }}) it else null } + ?.map { "%3s".format(it?.value ?: 0).replace(' ', '0') } + ?.joinToString(".") + + +class FinishBuildListener(pluginClassLoader: ClassLoader?, val startMemory: Long) : BuildAdapter() { val log = Logging.getLogger(this.javaClass) private var threadTracker: ThreadTracker? = ThreadTracker() + private val cleanup = CompilerServicesCleanup(pluginClassLoader) + override fun buildFinished(result: BuildResult?) { log.kotlinDebug("Build finished listener") - stopZipFileCache() - stopLowMemoryWatcher() - stopJobScheduler() - - // TODO: Try to clean up thread locals without this ugly hack - // TODO: Further investigation of PermGen leak (KT-6451) - removeThreadLocals() - - pluginClassLoader = null - result?.getGradle()?.removeListener(this) - - threadTracker?.checkThreadLeak(result?.getGradle()) - threadTracker = null - } - - public fun removeThreadLocals() { - try { - log.kotlinDebug("Remove ChildURLClassLoader thread locals") - - val thread = Thread.currentThread() - val threadLocalsField = javaClass().getDeclaredField("threadLocals") - threadLocalsField.setAccessible(true) - - val threadLocalMapClass = Class.forName("java.lang.ThreadLocal\$ThreadLocalMap") - val tableField = threadLocalMapClass.getDeclaredField("table") - tableField.setAccessible(true) - - val referentField = javaClass>().getDeclaredField("referent") - referentField.setAccessible(true) - - val table = tableField[threadLocalsField[thread]] as Array<*> - - for (entry in table) { - if (entry != null) { - val threadLocal = referentField[entry] as ThreadLocal<*>? - val classLoader = threadLocal?.javaClass?.getClassLoader() - if (classLoader is ParentLastURLClassLoader.ChildURLClassLoader) { - threadLocal?.remove() - } + val gradle = result?.getGradle() + if (gradle != null) { + // making cleanup only on recognized versions of gradle and if version < 2.4 + // otherwise it may cause problems e.g. with JobScheduler on subsequent runs + // this strategy may lead to memory leaks, but prevent crashes due to destroyed JobScheduler + // the reason for the strategy is the following: + // gradle < 2.4 has problems with plugin reuse in the daemon: new calls to the plugin are made with a new classloader + // for every new build. With statically initialized daemons like JobScheduler that leads to big leaks of classloaders and classes, + // therefore to reduce leaks JobScheduler (and deprecated ZipFileCache for now) should be stopped) + // It should be noted that because of this behavior there are no benefits of using daemon in these versions. + // Starting from 2.4 gradle using cached classloaders, that leads to effective class reusing in the daemon, but + // in that case premature stopping of the static daemons may lead to crashes. + comparableVersionStr(gradle.getGradleVersion())?.let { + log.kotlinDebug("detected gradle version $it") + if (it < comparableVersionStr("2.4")!!) { + cleanup() + // checking thread leaks only then cleaning up + threadTracker?.checkThreadLeak(gradle) } } - log.kotlinDebug("Removing ChildURLClassLoader thread locals finished successfully") - } catch (e: Throwable) { - log.kotlinDebug("Exception during thread locals remove: " + e) + threadTracker = null + gradle.removeListener(this) } + + // the value reported here is not necessarily a leak, since it is calculated before collecting the plugin classes + // but on subsequent runs in the daemon it should be rather small, then the classes are actually reused by the daemon (see above) + getUsedMemoryKb().let { log.kotlinDebug("[PERF] Used memory after build: $it kb (${"%+d".format(it-startMemory)} kb)") } + } +} + + +class CompilerServicesCleanup(private var pluginClassLoader: ClassLoader?) { + val log = Logging.getLogger(this.javaClass) + + fun invoke() { + assert(pluginClassLoader != null) + + log.kotlinDebug("compiler services cleanup") + + // \todo remove ZipFileCache cleanup after switching to recent idea libs + stopZipFileCache() + // currently unused since LowMemoryWatcher usages are (hopefully) all removed + // \todo doublecheck and remove LowMemoryWatcher cleanup code + //stopLowMemoryWatcher() + stopJobScheduler() + + pluginClassLoader = null } private fun stopZipFileCache() { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt index 8c7e18d7ab0..064143e0ac7 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/plugin/KotlinPluginWrapper.kt @@ -15,15 +15,16 @@ import java.lang.reflect.Method import org.jetbrains.kotlin.gradle.tasks.KotlinTasksProvider import java.net.URLClassLoader +// \todo simplify: the complicated structure is a leftover from dynamic loading of plugin core, could be significantly simplified now + abstract class KotlinBasePluginWrapper: Plugin { + val log = Logging.getLogger(this.javaClass) - companion object { - var pluginClassLoader: ClassLoader? = null - } - public override fun apply(project: Project) { + val startMemory = getUsedMemoryKb() + val sourceBuildScript = findSourceBuildScript(project) if (sourceBuildScript == null) { log.error("Failed to determine source cofiguration of kotlin plugin. Can not download core. Please verify that this or any parent project " + @@ -34,50 +35,14 @@ abstract class KotlinBasePluginWrapper: Plugin { val kotlinPluginVersion = loadKotlinVersionFromResource(log) project.getExtensions().getExtraProperties()?.set("kotlin.gradle.plugin.version", kotlinPluginVersion) - if (pluginClassLoader == null) - pluginClassLoader = createPluginIsolatedClassLoader(kotlinPluginVersion, sourceBuildScript) - else - log.kotlinDebug("Reusing classloader from previous run") - val plugin = getPlugin(pluginClassLoader!!, sourceBuildScript) + val plugin = getPlugin(this.javaClass.getClassLoader(), sourceBuildScript) plugin.apply(project) -// project.getGradle().addBuildListener(FinishBuildListener(pluginClassLoader)) + project.getGradle().addBuildListener(FinishBuildListener(this.javaClass.getClassLoader(), startMemory)) } protected abstract fun getPlugin(pluginClassLoader: ClassLoader, scriptHandler: ScriptHandler): Plugin - private fun createPluginClassLoader(projectVersion: String, sourceBuildScript: ScriptHandler): URLClassLoader { - val kotlinPluginDependencies: List = getPluginDependencies(projectVersion, sourceBuildScript) - log.kotlinDebug("Load plugin in regular URL classloader") - val kotlinPluginClassloader = URLClassLoader(kotlinPluginDependencies.toTypedArray(), this.javaClass.getClassLoader()) - - return kotlinPluginClassloader - } - - private fun createPluginIsolatedClassLoader(projectVersion: String, sourceBuildScript: ScriptHandler): ParentLastURLClassLoader { - val kotlinPluginDependencies: List = getPluginDependencies(projectVersion, sourceBuildScript) - log.kotlinDebug("Load plugin in parent-last URL classloader") - val kotlinPluginClassloader = ParentLastURLClassLoader(kotlinPluginDependencies, this.javaClass.getClassLoader()) - log.kotlinDebug("Class loader created") - - return kotlinPluginClassloader - } - - private fun getPluginDependencies(projectVersion: String, sourceBuildScript: ScriptHandler): List { - val dependencyHandler: DependencyHandler = sourceBuildScript.getDependencies() - val configurationsContainer: ConfigurationContainer = sourceBuildScript.getConfigurations() - - log.kotlinDebug("Creating configuration and dependency") - val kotlinPluginCoreCoordinates = "org.jetbrains.kotlin:kotlin-gradle-plugin-core:" + projectVersion - val dependency = dependencyHandler.create(kotlinPluginCoreCoordinates) - val configuration = configurationsContainer.detachedConfiguration(dependency) - - log.kotlinDebug("Resolving [" + kotlinPluginCoreCoordinates + "]") - val kotlinPluginDependencies: List = configuration.getResolvedConfiguration().getFiles({ true })!!.map { it.toURI().toURL() } - log.kotlinDebug("Resolved files: [" + kotlinPluginDependencies.toString() + "]") - return kotlinPluginDependencies - } - private fun findSourceBuildScript(project: Project): ScriptHandler? { log.kotlinDebug("Looking for proper script handler") var curProject = project @@ -111,4 +76,3 @@ open class Kotlin2JsPluginWrapper : KotlinBasePluginWrapper() { fun Logger.kotlinDebug(message: String) { this.debug("[KOTLIN] $message") } -