From c294a682dee86abbdbe80adf68891a84edf62ed0 Mon Sep 17 00:00:00 2001 From: Ilya Chernikov Date: Wed, 23 Sep 2015 12:55:18 +0200 Subject: [PATCH] Utils for checking presense of certain lines in a sequence, using this tool for checking log after daemon builds in tests, fixing explicit daemon shutdown used in the test, some minor fixes --- .../rmi/kotlinr/KotlinCompilerClient.kt | 8 +- .../org/jetbrains/kotlin/rmi/DaemonParams.kt | 2 + .../kotlin/rmi/service/CompileDaemon.kt | 21 +++-- .../kotlin/daemon/CompilerDaemonTest.kt | 67 ++++++++++---- .../jetbrains/kotlin/daemon/LogCheckUtil.kt | 91 +++++++++++++++++++ .../jps/build/SimpleKotlinJpsBuildTest.kt | 6 +- 6 files changed, 160 insertions(+), 35 deletions(-) create mode 100644 compiler/tests/org/jetbrains/kotlin/daemon/LogCheckUtil.kt diff --git a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt index cc7aa7f342d..2b115f861e9 100644 --- a/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt +++ b/compiler/rmi/kotlinr/src/org/jetbrains/kotlin/rmi/kotlinr/KotlinCompilerClient.kt @@ -93,14 +93,14 @@ public object KotlinCompilerClient { } - public fun shutdownCompileService(daemonOptions: DaemonOptions): Unit { - KotlinCompilerClient.connectToCompileService(CompilerId(), DaemonJVMOptions(), daemonOptions, DaemonReportingTargets(out = System.out), autostart = false, checkId = false) + public fun shutdownCompileService(compilerId: CompilerId, daemonOptions: DaemonOptions): Unit { + KotlinCompilerClient.connectToCompileService(compilerId, DaemonJVMOptions(), daemonOptions, DaemonReportingTargets(out = System.out), autostart = false, checkId = false) ?.shutdown() } - public fun shutdownCompileService(): Unit { - shutdownCompileService(DaemonOptions()) + public fun shutdownCompileService(compilerId: CompilerId): Unit { + shutdownCompileService(compilerId, DaemonOptions()) } diff --git a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt index 48e8f164c45..b081521d78c 100644 --- a/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt +++ b/compiler/rmi/rmi-interface/src/org/jetbrains/kotlin/rmi/DaemonParams.kt @@ -36,6 +36,7 @@ public val COMPILE_DAEMON_ENABLED_PROPERTY: String = "kotlin.daemon.enabled" public val COMPILE_DAEMON_JVM_OPTIONS_PROPERTY: String = "kotlin.daemon.jvm.options" public val COMPILE_DAEMON_OPTIONS_PROPERTY: String = "kotlin.daemon.options" public val COMPILE_DAEMON_CLIENT_ALIVE_PATH_PROPERTY: String = "kotlin.daemon.client.alive.path" +public val COMPILE_DAEMON_LOG_PATH_PROPERTY: String = "kotlin.daemon.log.path" public val COMPILE_DAEMON_REPORT_PERF_PROPERTY: String = "kotlin.daemon.perf" public val COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY: String = "kotlin.daemon.verbose" public val COMPILE_DAEMON_CMDLINE_OPTIONS_PREFIX: String = "--daemon-" @@ -299,6 +300,7 @@ public fun configureDaemonJVMOptions(opts: DaemonJVMOptions, inheritMemoryLimits System.getProperty(COMPILE_DAEMON_REPORT_PERF_PROPERTY)?.let { opts.jvmParams.add("D" + COMPILE_DAEMON_REPORT_PERF_PROPERTY) } System.getProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY)?.let { opts.jvmParams.add("D" + COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) } + System.getProperty(COMPILE_DAEMON_LOG_PATH_PROPERTY)?.let { opts.jvmParams.add("D$COMPILE_DAEMON_LOG_PATH_PROPERTY=$it" ) } return opts } diff --git a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt index a8354de5a8f..201d5498f00 100644 --- a/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt +++ b/compiler/rmi/rmi-server/src/org/jetbrains/kotlin/rmi/service/CompileDaemon.kt @@ -59,18 +59,19 @@ class LogStream(name: String) : OutputStream() { public object CompileDaemon { init { - val logPath: String = System.getProperty("kotlin.daemon.log.path")?.trimEnd('/','\\') ?: "%t" val logTime: String = SimpleDateFormat("yyyy-MM-dd.HH-mm-ss-SSS").format(Date()) + val (logPath: String, fileIsGiven: Boolean) = + System.getProperty(COMPILE_DAEMON_LOG_PATH_PROPERTY)?.let { Pair(it, File(it).isFile) } ?: Pair("%t", false) val cfg: String = - "handlers = java.util.logging.FileHandler\n" + - "java.util.logging.FileHandler.level = ALL\n" + - "java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" + - "java.util.logging.FileHandler.encoding = UTF-8\n" + - "java.util.logging.FileHandler.limit = 1073741824\n" + // 1Mb - "java.util.logging.FileHandler.count = 3\n" + - "java.util.logging.FileHandler.append = false\n" + - "java.util.logging.FileHandler.pattern = $logPath/$COMPILE_DAEMON_DEFAULT_FILES_PREFIX.$logTime.%u%g.log\n" + - "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s\\n\n" + "handlers = java.util.logging.FileHandler\n" + + "java.util.logging.FileHandler.level = ALL\n" + + "java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter\n" + + "java.util.logging.FileHandler.encoding = UTF-8\n" + + "java.util.logging.FileHandler.limit = ${if (fileIsGiven) 0 else (1 shl 20)}\n" + // if file is provided - disabled, else - 1Mb + "java.util.logging.FileHandler.count = ${if (fileIsGiven) 1 else 3}\n" + + "java.util.logging.FileHandler.append = $fileIsGiven\n" + + "java.util.logging.FileHandler.pattern = ${if (fileIsGiven) logPath else (logPath + File.separator + "$COMPILE_DAEMON_DEFAULT_FILES_PREFIX.$logTime.%u%g.log")}\n" + + "java.util.logging.SimpleFormatter.format = %1\$tF %1\$tT.%1\$tL [%3\$s] %4\$s: %5\$s\\n\n" LogManager.getLogManager().readConfiguration(cfg.byteInputStream()) } diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt index 8ac0f4709fd..96b306070e0 100644 --- a/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt +++ b/compiler/tests/org/jetbrains/kotlin/daemon/CompilerDaemonTest.kt @@ -31,7 +31,6 @@ public class CompilerDaemonTest : KotlinIntegrationTestBase() { data class CompilerResults(val resultCode: Int, val out: String) val daemonOptions by lazy(LazyThreadSafetyMode.NONE) { DaemonOptions(runFilesPath = tmpdir.absolutePath) } - val daemonJVMOptions by lazy(LazyThreadSafetyMode.NONE) { DaemonJVMOptions() } val compilerId by lazy(LazyThreadSafetyMode.NONE) { CompilerId.makeCompilerId( File(KotlinIntegrationTestBase.getCompilerLib(), "kotlin-compiler.jar"), @@ -39,33 +38,52 @@ public class CompilerDaemonTest : KotlinIntegrationTestBase() { File("dependencies/bootstrap-compiler/Kotlin/kotlinc/lib/kotlin-reflect.jar")) } - private fun compileOnDaemon(args: Array): CompilerResults { - System.setProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY, "") - try { - val daemon = KotlinCompilerClient.connectToCompileService(compilerId, daemonJVMOptions, daemonOptions, DaemonReportingTargets(out = System.err), autostart = true, checkId = true) - TestCase.assertNotNull("failed to connect daemon", daemon) - val strm = ByteArrayOutputStream() - val code = KotlinCompilerClient.compile(daemon!!, args, strm) - return CompilerResults(code, strm.toString()) - } - finally { - System.clearProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) - } + private fun compileOnDaemon(daemonJVMOptions: DaemonJVMOptions, args: Array): CompilerResults { + val daemon = KotlinCompilerClient.connectToCompileService(compilerId, daemonJVMOptions, daemonOptions, DaemonReportingTargets(out = System.err), autostart = true, checkId = true) + TestCase.assertNotNull("failed to connect daemon", daemon) + val strm = ByteArrayOutputStream() + val code = KotlinCompilerClient.compile(daemon!!, args, strm) + return CompilerResults(code, strm.toString()) } private fun runDaemonCompilerTwice(logName: String, vararg arguments: String): Unit { - KotlinCompilerClient.shutdownCompileService(daemonOptions) + KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) + + System.setProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY, "") + val logFile = File.createTempFile("kotlin-daemon-test.", ".log") + System.setProperty(COMPILE_DAEMON_LOG_PATH_PROPERTY, logFile.absolutePath) + + val daemonJVMOptions = configureDaemonJVMOptions(false) + var daemonShotDown = false +// daemonJVMOptions.jvmParams.add("agentlib:jdwp=transport=dt_socket\\,server=y\\,suspend=y\\,address=5005") try { - val res1 = compileOnDaemon(arguments) + val res1 = compileOnDaemon(daemonJVMOptions, arguments) TestCase.assertEquals("first compilation failed:\n${res1.out}", 0, res1.resultCode) - val res2 = compileOnDaemon(arguments) + val res2 = compileOnDaemon(daemonJVMOptions, arguments) TestCase.assertEquals("second compilation failed:\n${res2.out}", 0, res2.resultCode) TestCase.assertEquals("build results differ", CliBaseTest.removePerfOutput(res1.out), CliBaseTest.removePerfOutput(res2.out)) + KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) + daemonShotDown = true + logFile.reader().useLines { + it.ifNotContainsSequence( LinePattern("Kotlin compiler daemon version"), + LinePattern("Starting compilation with args: "), + LinePattern("Starting compilation with args: "), + LinePattern("Shutdown complete")) + { (unmatchedPattern, lineNo) -> + TestCase.fail("pattern not found in the input: " + unmatchedPattern.regex + + "\nunmatched part of the log file (" + logFile.absolutePath + + ") from line " + lineNo + ":\n\n" + logFile.reader().useLines { it.drop(lineNo).joinToString("\n") }) + } + } + logFile.delete() // TODO: add performance comparison assert } finally { - KotlinCompilerClient.shutdownCompileService(daemonOptions) + if (!daemonShotDown) + KotlinCompilerClient.shutdownCompileService(compilerId, daemonOptions) + System.clearProperty(COMPILE_DAEMON_LOG_PATH_PROPERTY) + System.clearProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) } } @@ -91,12 +109,12 @@ public class CompilerDaemonTest : KotlinIntegrationTestBase() { TestCase.assertEquals(arrayListOf("aaa", "bbb,ccc", "ddd", "xxx,yyy"), opts.jvmParams) } finally { - backupJvmOptions?.let { System.setProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY, it) } + restoreSystemProperty(COMPILE_DAEMON_JVM_OPTIONS_PROPERTY, backupJvmOptions) } } public fun testDaemonOptionsParsing() { - val backupJvmOptions = System.getProperty(COMPILE_DAEMON_OPTIONS_PROPERTY) + val backupOptions = System.getProperty(COMPILE_DAEMON_OPTIONS_PROPERTY) try { System.setProperty(COMPILE_DAEMON_OPTIONS_PROPERTY, "runFilesPath=abcd,clientAliveFlagPath=efgh,autoshutdownIdleSeconds=1111") val opts = configureDaemonOptions() @@ -105,7 +123,16 @@ public class CompilerDaemonTest : KotlinIntegrationTestBase() { TestCase.assertEquals(1111, opts.autoshutdownIdleSeconds) } finally { - backupJvmOptions?.let { System.setProperty(COMPILE_DAEMON_OPTIONS_PROPERTY, it) } + restoreSystemProperty(COMPILE_DAEMON_OPTIONS_PROPERTY, backupOptions) } } } + +fun restoreSystemProperty(propertyName: String, backupValue: String?) { + if (backupValue == null) { + System.clearProperty(propertyName) + } + else { + System.setProperty(propertyName, backupValue) + } +} diff --git a/compiler/tests/org/jetbrains/kotlin/daemon/LogCheckUtil.kt b/compiler/tests/org/jetbrains/kotlin/daemon/LogCheckUtil.kt new file mode 100644 index 00000000000..6b04f1f4c40 --- /dev/null +++ b/compiler/tests/org/jetbrains/kotlin/daemon/LogCheckUtil.kt @@ -0,0 +1,91 @@ +/* + * Copyright 2010-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.daemon + +import kotlin.text.MatchResult +import kotlin.text.Regex + +/** + * holder of a [regex] and optional [matchCheck] for additional checks on match result + */ +internal class LinePattern(val regex: Regex, val matchCheck: (MatchResult) -> Boolean = { true }) +internal fun LinePattern(regex: String, matchCheck: (MatchResult) -> Boolean = { true }) = LinePattern(regex.toRegex(), matchCheck) + +/** + * calls [body] if receiver does not contain complete sequence of lines matched by [patternsIter], separated by any number of other lines + * [body] receives first unmatched pattern and index of last matched line in the sequence + */ +internal fun Sequence.ifNotContainsSequence(patternsIter: Iterator, + body: (LinePattern, Int) -> Unit) : Unit { + class Accumulator(it: Iterator) { + val iter = EndBoundIteratorWithValue(it) + var lineNo = 1 + var lastMatchedLineNo = 0 + fun nextLineAndPattern(): Accumulator { iter.traverseNext(); lastMatchedLineNo = lineNo; return nextLine() } + fun nextLine(): Accumulator { lineNo++; return this } + } + val res = fold(Accumulator(patternsIter)) + { acc, line -> + when { + !acc.iter.isValid() -> return@fold acc + acc.iter.value.regex.match(line)?.let { acc.iter.value.matchCheck(it) } ?: false -> acc.nextLineAndPattern() + else -> acc.nextLine() + } + } + if (res.iter.isValid()) { + body(res.iter.value, res.lastMatchedLineNo) + } +} + + +/** + * calls [body] if receiver does not contain complete sequence of lines matched by [patterns], separated by any number of other lines + * [body] receives first unmatched pattern and index of last matched line in the sequence + */ +internal fun Sequence.ifNotContainsSequence(patterns: List, + body: (LinePattern, Int) -> Unit): Unit { + ifNotContainsSequence(patterns.iterator(), body) +} + + +/** + * calls [body] if receiver does not contain complete sequence of lines matched by [patterns], separated by any number of other lines + * [body] receives first unmatched pattern and index of last matched line in the sequence + */ +internal fun Sequence.ifNotContainsSequence(vararg patterns: LinePattern, + body: (LinePattern, Int) -> Unit): Unit { + ifNotContainsSequence(patterns.iterator(), body) +} + + +// emulates Stepanov's / STL iterator, but with "embedded" end check via isValid: +// iterator points to a current value and upon init points to the first element or is invalid +// allows to express some algorithms more concisely +private class EndBoundIteratorWithValue>(val base: Iter) { + private var _value: T? = base.nextOrNull() + + val value: T get() = _value ?: throw Exception("Dereferencing invalid iterator") + + fun isValid(): Boolean = _value != null + + fun traverseNext(): EndBoundIteratorWithValue { + _value = base.nextOrNull() + return this + } +} + +private fun Iterator.nextOrNull(): T? = if (hasNext()) next() else null diff --git a/jps-plugin/test/org/jetbrains/kotlin/jps/build/SimpleKotlinJpsBuildTest.kt b/jps-plugin/test/org/jetbrains/kotlin/jps/build/SimpleKotlinJpsBuildTest.kt index 3b1a7f395b5..f32820d9b48 100644 --- a/jps-plugin/test/org/jetbrains/kotlin/jps/build/SimpleKotlinJpsBuildTest.kt +++ b/jps-plugin/test/org/jetbrains/kotlin/jps/build/SimpleKotlinJpsBuildTest.kt @@ -20,6 +20,7 @@ import com.intellij.util.PathUtil import org.jetbrains.jps.model.java.JpsJavaExtensionService import org.jetbrains.kotlin.rmi.COMPILE_DAEMON_CLIENT_ALIVE_PATH_PROPERTY import org.jetbrains.kotlin.rmi.COMPILE_DAEMON_ENABLED_PROPERTY +import org.jetbrains.kotlin.rmi.COMPILE_DAEMON_LOG_PATH_PROPERTY import org.jetbrains.kotlin.rmi.COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY import org.jetbrains.kotlin.test.JetTestUtils import java.io.File @@ -71,12 +72,15 @@ public class SimpleKotlinJpsBuildTest : AbstractKotlinJpsBuildTestCase() { System.setProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY, "") // spaces in the name to test proper file name handling val flagFile = File.createTempFile("kotlin-jps - tests-", "-is-running"); + val logFile = File.createTempFile("kotlin-daemon", ".log") + System.setProperty(COMPILE_DAEMON_CLIENT_ALIVE_PATH_PROPERTY, flagFile.absolutePath) + System.setProperty(COMPILE_DAEMON_LOG_PATH_PROPERTY, logFile.absolutePath) try { - System.setProperty(COMPILE_DAEMON_CLIENT_ALIVE_PATH_PROPERTY, flagFile.absolutePath) testLoadingKotlinFromDifferentModules() } finally { flagFile.delete() + System.clearProperty(COMPILE_DAEMON_LOG_PATH_PROPERTY) System.clearProperty(COMPILE_DAEMON_CLIENT_ALIVE_PATH_PROPERTY) System.clearProperty(COMPILE_DAEMON_VERBOSE_REPORT_PROPERTY) System.clearProperty(COMPILE_DAEMON_ENABLED_PROPERTY)