azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce other JVM options inheritance for daemon

Browse Source
This commit is contained in:
Ilya Chernikov
2017-05-04 23:15:44 +02:00
parent 6190b1ec2e
commit 9d2ae54d2c
9 changed files with 110 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/compiler-runner/src/org/jetbrains/kotlin/compilerRunner/KotlinCompilerRunner.kt
+1 -1
View File
@@ -60,7 +60,7 @@ abstract class KotlinCompilerRunner<in Env : CompilerEnvironment> {
environment: Env,
daemonOptions: DaemonOptions = configureDaemonOptions()
): CompileServiceSession? {
val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritAdditionalProperties = true)
val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritOtherJvmOptions = false, inheritAdditionalProperties = true)
val daemonReportMessages = ArrayList<DaemonReportMessage>()
val daemonReportingTargets = DaemonReportingTargets(messages = daemonReportMessages)
compiler/daemon/daemon-client/src/org/jetbrains/kotlin/daemon/client/KotlinCompilerClient.kt
+15 -13
View File
@@ -102,7 +102,7 @@ object KotlinCompilerClient {
autostart: Boolean,
leaseSession: Boolean,
sessionAliveFlagFile: File? = null
): CompileServiceSession? = connectLoop(reportingTargets) {
): CompileServiceSession? = connectLoop(reportingTargets, autostart) { isLastAttempt ->
ensureServerHostnameIsSetUp()
val (service, newJVMOptions) = tryFindSuitableDaemonOrNewOpts(File(daemonOptions.runFilesPath), compilerId, daemonJVMOptions, { cat, msg -> reportingTargets.report(cat, msg) })
if (service != null) {
@@ -119,7 +119,7 @@ object KotlinCompilerClient {
}
} else {
reportingTargets.report(DaemonReportCategory.DEBUG, "no suitable daemon found")
if (autostart) {
if (!isLastAttempt && autostart) {
startDaemon(compilerId, newJVMOptions, daemonOptions, reportingTargets)
reportingTargets.report(DaemonReportCategory.DEBUG, "new daemon started, trying to find it")
}
@@ -236,7 +236,7 @@ object KotlinCompilerClient {
fun main(vararg args: String) {
val compilerId = CompilerId()
val daemonOptions = configureDaemonOptions()
val daemonLaunchingOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritAdditionalProperties = true)
val daemonLaunchingOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritOtherJvmOptions = false, inheritAdditionalProperties = true)
val clientOptions = configureClientOptions()
val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, daemonLaunchingOptions, clientOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX)
@@ -311,24 +311,26 @@ object KotlinCompilerClient {
// --- Implementation ---------------------------------------
@Synchronized
private inline fun <R> connectLoop(reportingTargets: DaemonReportingTargets, body: () -> R?): R? {
private inline fun <R> connectLoop(reportingTargets: DaemonReportingTargets, autostart: Boolean, body: (Boolean) -> R?): R? {
try {
var attempts = 0
while (attempts < DAEMON_CONNECT_CYCLE_ATTEMPTS) {
attempts += 1
var attempts = 1
while (true) {
val (res, err) = try {
body() to null
body(attempts >= DAEMON_CONNECT_CYCLE_ATTEMPTS) to null
}
catch (e: SocketException) { null to e }
catch (e: ConnectException) { null to e }
catch (e: ConnectIOException) { null to e }
catch (e: UnmarshalException) { null to e }
when {
res != null -> return res
err != null && attempts >= DAEMON_CONNECT_CYCLE_ATTEMPTS -> throw err
err != null ->
reportingTargets.report(DaemonReportCategory.INFO, "retrying($attempts) on: " + err.toString())
if (res != null) return res
reportingTargets.report(DaemonReportCategory.INFO,
(if (attempts >= DAEMON_CONNECT_CYCLE_ATTEMPTS || !autostart) "no more retries on: " else "retrying($attempts) on: ")
+ err?.toString())
if (attempts++ > DAEMON_CONNECT_CYCLE_ATTEMPTS || !autostart) {
return null
}
}
}
compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/ClientUtils.kt
+4
View File
@@ -44,6 +44,10 @@ private const val ORPHANED_RUN_FILE_AGE_THRESHOLD_MS = 1000000L
data class DaemonWithMetadata(val daemon: CompileService, val runFile: File, val jvmOptions: DaemonJVMOptions)
// TODO: write metadata into discovery file to speed up selection
// TODO: consider using compiler jar signature (checksum) as a CompilerID (plus java version, plus ???) instead of classpath checksum
// would allow to use same compiler from taken different locations
// reqs: check that plugins (or anything els) should not be part of the CP
fun walkDaemons(registryDir: File,
compilerId: CompilerId,
fileToCompareTimestamp: File,
compiler/daemon/daemon-common/src/org/jetbrains/kotlin/daemon/common/DaemonParams.kt
+40 -6
View File
@@ -257,15 +257,44 @@ data class CompilerId(
fun isDaemonEnabled(): Boolean = System.getProperty(COMPILE_DAEMON_ENABLED_PROPERTY) != null
fun configureDaemonJVMOptions(opts: DaemonJVMOptions,
vararg additionalParams: String,
inheritMemoryLimits: Boolean,
inheritOtherJvmOptions: Boolean,
inheritAdditionalProperties: Boolean
): DaemonJVMOptions =
configureDaemonJVMOptions(opts, additionalParams.asIterable(), inheritMemoryLimits, inheritOtherJvmOptions, inheritAdditionalProperties)
// TODO: expose sources for testability and test properly
fun configureDaemonJVMOptions(opts: DaemonJVMOptions,
additionalParams: Iterable<String>,
inheritMemoryLimits: Boolean,
inheritOtherJvmOptions: Boolean,
inheritAdditionalProperties: Boolean
): DaemonJVMOptions {
// note: sequence matters, explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing
if (inheritMemoryLimits) {
ManagementFactory.getRuntimeMXBean().inputArguments.filterExtractProps(opts.mappers, "-")
// note: sequence matters, e.g. explicit override in COMPILE_DAEMON_JVM_OPTIONS_PROPERTY should be done after inputArguments processing
if (inheritMemoryLimits || inheritOtherJvmOptions) {
val jvmArguments = ManagementFactory.getRuntimeMXBean().inputArguments
val targetOptions = if (inheritMemoryLimits) opts else DaemonJVMOptions()
val otherArgs = jvmArguments.filterExtractProps(targetOptions.mappers, prefix = "-")
if (inheritMemoryLimits && opts.maxMemory.isBlank()) {
val maxMemBytes = Runtime.getRuntime().maxMemory()
// rounding up
val maxMemMegabytes = maxMemBytes / (1024 * 1024) + if (maxMemBytes % (1024 * 1024) == 0L) 0 else 1
opts.maxMemory = "${maxMemMegabytes}m"
}
if (inheritOtherJvmOptions) {
opts.jvmParams.addAll(
otherArgs.filterNot {
it.startsWith("agentlib") ||
it.startsWith("D" + COMPILE_DAEMON_LOG_PATH_PROPERTY) ||
it.startsWith("D" + KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY) ||
it.startsWith("D" + COMPILE_DAEMON_JVM_OPTIONS_PROPERTY) ||
it.startsWith("D" + COMPILE_DAEMON_OPTIONS_PROPERTY)
})
}
}
System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)?.let {
opts.jvmParams.addAll(
@@ -275,7 +304,10 @@ fun configureDaemonJVMOptions(opts: DaemonJVMOptions,
.filterExtractProps(opts.mappers, "-", opts.restMapper))
}
// assuming that from the conflicting options the last one is taken
// TODO: compare and override
opts.jvmParams.addAll(additionalParams)
if (inheritAdditionalProperties) {
System.getProperty(COMPILE_DAEMON_LOG_PATH_PROPERTY)?.let { opts.jvmParams.add("D$COMPILE_DAEMON_LOG_PATH_PROPERTY=\"$it\"") }
System.getProperty(KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY)?.let { opts.jvmParams.add("D$KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY") }
@@ -285,12 +317,14 @@ fun configureDaemonJVMOptions(opts: DaemonJVMOptions,
fun configureDaemonJVMOptions(vararg additionalParams: String,
inheritMemoryLimits: Boolean,
inheritAdditionalProperties: Boolean
inheritMemoryLimits: Boolean,
inheritOtherJvmOptions: Boolean,
inheritAdditionalProperties: Boolean
): DaemonJVMOptions =
configureDaemonJVMOptions(DaemonJVMOptions(),
additionalParams = *additionalParams,
inheritMemoryLimits = inheritMemoryLimits,
inheritOtherJvmOptions = inheritOtherJvmOptions,
inheritAdditionalProperties = inheritAdditionalProperties)
compiler/daemon/src/org/jetbrains/kotlin/daemon/CompileServiceImpl.kt
+4 -2
View File
@@ -107,6 +107,8 @@ class CompileServiceImpl(
val onShutdown: () -> Unit
) : CompileService {
private val log by lazy { Logger.getLogger("compiler") }
init {
System.setProperty(KOTLIN_COMPILER_ENVIRONMENT_KEEPALIVE_PROPERTY, "true")
}
@@ -184,8 +186,6 @@ class CompileServiceImpl(
@Volatile private var _lastUsedSeconds = nowSeconds()
val lastUsedSeconds: Long get() = if (rwlock.isWriteLocked || rwlock.readLockCount - rwlock.readHoldCount > 0) nowSeconds() else _lastUsedSeconds
private val log by lazy { Logger.getLogger("compiler") }
private val rwlock = ReentrantReadWriteLock()
private var runFile: File
@@ -212,6 +212,8 @@ class CompileServiceImpl(
}
override fun getDaemonJVMOptions(): CompileService.CallResult<DaemonJVMOptions> = ifAlive {
log.info("getDaemonJVMOptions: $daemonJVMOptions")// + daemonJVMOptions.mappers.flatMap { it.toArgs("-") })
CompileService.CallResult.Good(daemonJVMOptions)
}
compiler/daemon/src/org/jetbrains/kotlin/daemon/KotlinCompileDaemon.kt
+6 -2
View File
@@ -91,8 +91,10 @@ object KotlinCompileDaemon {
fun main(args: Array<String>) {
ensureServerHostnameIsSetUp()
val jvmArguments = ManagementFactory.getRuntimeMXBean().inputArguments
log.info("Kotlin compiler daemon version " + (loadVersionFromResource() ?: "<unknown>"))
log.info("daemon JVM args: " + ManagementFactory.getRuntimeMXBean().inputArguments.joinToString(" "))
log.info("daemon JVM args: " + jvmArguments.joinToString(" "))
log.info("daemon args: " + args.joinToString(" "))
setIdeaIoUseFallback()
@@ -101,7 +103,9 @@ object KotlinCompileDaemon {
val daemonOptions = DaemonOptions()
try {
val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true, inheritAdditionalProperties = true)
val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = true,
inheritOtherJvmOptions = true,
inheritAdditionalProperties = true)
val filteredArgs = args.asIterable().filterExtractProps(compilerId, daemonOptions, prefix = COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX)
compiler/tests/org/jetbrains/kotlin/daemon/CompilerApiTest.kt
+9 -2
View File
@@ -128,7 +128,7 @@ class CompilerApiTest : KotlinIntegrationTestBase() {
val logFile = createTempFile("kotlin-daemon-test.", ".log")
val daemonJVMOptions = configureDaemonJVMOptions("D$COMPILE_DAEMON_LOG_PATH_PROPERTY=\"${logFile.loggerCompatiblePath}\"",
inheritMemoryLimits = false, inheritAdditionalProperties = false)
inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false)
val jar = tmpdir.absolutePath + File.separator + "hello.jar"
try {
@@ -169,7 +169,7 @@ class CompilerApiTest : KotlinIntegrationTestBase() {
val logFile = createTempFile("kotlin-daemon-test.", ".log")
val daemonJVMOptions = configureDaemonJVMOptions("D$COMPILE_DAEMON_LOG_PATH_PROPERTY=\"${logFile.loggerCompatiblePath}\"",
inheritMemoryLimits = false, inheritAdditionalProperties = false)
inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false)
try {
val (code, outputs) = compileOnDaemon(
flagFile, compilerId, daemonJVMOptions, daemonOptions, TestMessageCollector(),
@@ -203,3 +203,10 @@ class TestMessageCollector : MessageCollector {
override fun hasErrors(): Boolean = messages.any { it.severity == CompilerMessageSeverity.EXCEPTION || it.severity == CompilerMessageSeverity.ERROR }
}
fun TestMessageCollector.assertHasMessage(msg: String) {
assert(messages.any { it.message.contains(msg) }) {
"Expecting message \"$msg\", actual:\n${messages.joinToString("\n") { it.message }}"
}
}
compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt
+30 -6
View File
@@ -97,7 +97,8 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
baseOpts,
*additionalArgs.toTypedArray(),
inheritMemoryLimits = xmx > 0,
inheritAdditionalProperties = false)
inheritAdditionalProperties = false,
inheritOtherJvmOptions = false)
}
fun testHelloApp() {
@@ -144,14 +145,14 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
val backupJvmOptions = System.getProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)
try {
System.setProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY, "-aaa,-bbb\\,ccc,-ddd,-Xmx200m,-XX:MaxPermSize=10k,-XX:ReservedCodeCacheSize=100,-xxx\\,yyy")
val opts = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false)
val opts = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false, inheritOtherJvmOptions = false)
assertEquals("200m", opts.maxMemory)
assertEquals("10k", opts.maxPermSize)
assertEquals("100", opts.reservedCodeCacheSize)
assertEquals(arrayListOf("aaa", "bbb,ccc", "ddd", "xxx,yyy"), opts.jvmParams)
System.setProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY, "-Xmx300m,-XX:MaxPermSize=10k,-XX:ReservedCodeCacheSize=100")
val opts2 = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false)
val opts2 = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false, inheritOtherJvmOptions = false)
assertEquals("300m", opts2.maxMemory)
assertEquals( -1, DaemonJVMOptionsMemoryComparator().compare(opts, opts2))
assertEquals("300m", listOf(opts, opts2).maxWith(DaemonJVMOptionsMemoryComparator())?.maxMemory)
@@ -161,6 +162,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
val myXmxVal = myXmxParam.substring(4)
System.clearProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY)
val opts3 = configureDaemonJVMOptions(inheritMemoryLimits = true,
inheritOtherJvmOptions = true,
inheritAdditionalProperties = false)
assertEquals(myXmxVal, opts3.maxMemory)
}
@@ -229,7 +231,7 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions)
val daemonJVMOptions = configureDaemonJVMOptions("-abracadabra", inheritMemoryLimits = false, inheritAdditionalProperties = false)
val daemonJVMOptions = configureDaemonJVMOptions("-abracadabra", inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false)
val messageCollector = TestMessageCollector()
@@ -238,8 +240,30 @@ class CompilerDaemonTest : KotlinIntegrationTestBase() {
assertNull(daemon)
assertTrue("Expecting error message, actual:\n${messageCollector.messages.joinToString("\n") { it.message }}",
messageCollector.messages.any { it.message == "Unrecognized option: --abracadabra" })
messageCollector.assertHasMessage("Unrecognized option: --abracadabra")
}
}
// TODO: find out how to reliably cause the retry
fun ignore_testDaemonStartRetry() {
withFlagFile(getTestName(true), ".alive") { flagFile ->
val daemonOptions = DaemonOptions(shutdownDelayMilliseconds = 1, verbose = true, runFilesPath = File(tmpdir, getTestName(true)).absolutePath)
KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions)
val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false)
val messageCollector = TestMessageCollector()
val daemon = KotlinCompilerClient.connectToCompileService(compilerId, flagFile, daemonJVMOptions, daemonOptions,
DaemonReportingTargets(messageCollector = messageCollector), autostart = true)
assertNull(daemon)
messageCollector.assertHasMessage("retrying(0) on:")
messageCollector.assertHasMessage("retrying(1) on:")
// TODO: messageCollector.assertHasNoMessage("retrying(2) on:")
messageCollector.assertHasMessage("no more retries on:")
}
}
plugins/source-sections/source-sections-compiler/tests/org/jetbrains/kotlin/sourceSections/SourceSectionsTest.kt
+1 -1
View File
@@ -196,7 +196,7 @@ class SourceSectionsTest : TestCaseWithTmpdir() {
withFlagFile("sourceSections", ".alive") { aliveFile ->
val daemonOptions = DaemonOptions(runFilesPath = File(tmpdir, getTestName(true)).absolutePath, verbose = true, reportPerf = true)
val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritAdditionalProperties = false)
val daemonJVMOptions = configureDaemonJVMOptions(inheritMemoryLimits = false, inheritOtherJvmOptions = false, inheritAdditionalProperties = false)
val messageCollector = TestMessageCollector()
val daemonWithSession = KotlinCompilerClient.connectAndLease(compilerId, aliveFile, daemonJVMOptions, daemonOptions,