[Native][tests] Add an ability to fail tests on exceeded timeout

native/native.tests/build.gradle.kts

@@ -46,7 +46,8 @@ enum class TestProperty(shortName: String) {
     KOTLIN_NATIVE_HOME("nativeHome"),
     COMPILER_CLASSPATH("compilerClasspath"),
     TEST_MODE("mode"),
-    USE_CACHE("useCache");
+    USE_CACHE("useCache"),
+    EXECUTION_TIMEOUT("executionTimeout");
 
     private val propertyName = "kotlin.internal.native.test.$shortName"
 
@@ -81,6 +82,7 @@ if (kotlinBuildProperties.isKotlinNativeEnabled) {
         // Pass Gradle properties as JVM properties so test process can read them.
         TestProperty.TEST_MODE.setUpFromGradleProperty(this)
         TestProperty.USE_CACHE.setUpFromGradleProperty(this)
+        TestProperty.EXECUTION_TIMEOUT.setUpFromGradleProperty(this)
 
         useJUnitPlatform()
     }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/LoggedData.kt

@@ -6,7 +6,10 @@
 package org.jetbrains.kotlin.konan.blackboxtest.support
 
 import org.jetbrains.kotlin.cli.common.ExitCode
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.AbstractRunner
 import java.io.File
+import kotlin.time.Duration
+import kotlin.time.DurationUnit
 
 /**
  * A piece of information that makes sense for the user, and that can be logged to a file or
@@ -19,6 +22,12 @@ internal abstract class LoggedData {
     protected abstract fun computeText(): String
     final override fun toString() = text
 
+    fun withErrorMessageHeader(errorMessageHeader: String): String = buildString {
+        appendLine(errorMessageHeader)
+        appendLine()
+        appendLine(this@LoggedData)
+    }
+
     class CompilerParameters(
         private val compilerArgs: Array<String>,
         private val sourceModules: Collection<TestModule>
@@ -42,7 +51,7 @@ internal abstract class LoggedData {
         private val exitCode: ExitCode,
         private val compilerOutput: String,
         private val compilerOutputHasErrors: Boolean,
-        private val durationMillis: Long
+        private val duration: Duration
     ) : LoggedData() {
         override fun computeText(): String {
             val problems = listOfNotNull(
@@ -59,7 +68,7 @@ internal abstract class LoggedData {
 
                 appendLine("COMPILER CALL:")
                 appendLine("- Exit code: ${exitCode.code} (${exitCode.name})")
-                appendDuration(durationMillis)
+                appendDuration(duration)
                 appendLine()
                 appendLine("========== BEGIN: RAW COMPILER OUTPUT ==========")
                 if (compilerOutput.isNotEmpty()) appendLine(compilerOutput.trimEnd())
@@ -100,22 +109,19 @@ internal abstract class LoggedData {
 
     class TestRun(
         private val parameters: TestRunParameters,
-        private val exitCode: Int,
-        private val stdOut: String,
-        private val stdErr: String,
-        private val durationMillis: Long
+        private val runResult: AbstractRunner.RunResult.Completed
     ) : LoggedData() {
         override fun computeText() = buildString {
             appendLine("TEST RUN:")
-            appendLine("- Exit code: $exitCode")
-            appendDuration(durationMillis)
+            appendLine("- Exit code: ${runResult.exitCode}")
+            appendDuration(runResult.duration)
             appendLine()
             appendLine("========== BEGIN: TEST STDOUT ==========")
-            if (stdOut.isNotEmpty()) appendLine(stdOut.trimEnd())
+            if (runResult.stdOut.isNotEmpty()) appendLine(runResult.stdOut.trimEnd())
             appendLine("========== END: TEST STDOUT ==========")
             appendLine()
             appendLine("========== BEGIN: TEST STDERR ==========")
-            if (stdErr.isNotEmpty()) appendLine(stdErr.trimEnd())
+            if (runResult.stdErr.isNotEmpty()) appendLine(runResult.stdErr.trimEnd())
             appendLine("========== END: TEST STDERR ==========")
             appendLine()
             appendLine(parameters)
@@ -139,15 +145,27 @@ internal abstract class LoggedData {
         }
     }
 
+    class TestRunTimeoutExceeded(parameters: TestRunParameters, timeout: Duration) : TimeoutExceeded(parameters, timeout)
+
+    abstract class TimeoutExceeded(
+        private val parameters: LoggedData,
+        private val timeout: Duration
+    ) : LoggedData() {
+        override fun computeText() = buildString {
+            appendLine("TIMED OUT:")
+            appendLine("- Max permitted duration: $timeout")
+            appendLine()
+            appendLine(parameters)
+        }
+    }
+
     companion object {
-        @JvmStatic
         protected fun StringBuilder.appendList(header: String, list: Iterable<Any?>): StringBuilder {
             appendLine(header)
             list.forEach(::appendLine)
             return this
         }
 
-        @JvmStatic
         protected fun StringBuilder.appendArguments(header: String, args: Iterable<String>): StringBuilder {
             appendLine(header)
 
@@ -171,7 +189,7 @@ internal abstract class LoggedData {
             return this
         }
 
-        protected fun StringBuilder.appendDuration(durationMillis: Long): StringBuilder =
-            append("- Duration: ").append(String.format("%.2f", durationMillis.toDouble() / 1000)).appendLine(" seconds")
+        protected fun StringBuilder.appendDuration(duration: Duration): StringBuilder =
+            append("- Duration: ").appendLine(duration.toString(DurationUnit.SECONDS, 2))
     }
 }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCompiler.kt

@@ -17,7 +17,9 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestModule.Companion.allF
 import org.jetbrains.kotlin.konan.blackboxtest.support.util.*
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
 import java.io.*
-import kotlin.system.measureTimeMillis
+import kotlin.time.Duration
+import kotlin.time.ExperimentalTime
+import kotlin.time.measureTime
 
 internal class TestCompilationFactory(private val environment: TestEnvironment) {
     private val cachedCompilations = ThreadSafeCache<TestCompilationCacheKey, TestCompilation>()
@@ -175,14 +177,12 @@ internal sealed interface TestCompilationResult {
             is DependencyFailures -> fail { describeDependencyFailures() }
         }
 
-        private fun Failure.describeFailure() = buildString {
+        private fun Failure.describeFailure() = loggedData.withErrorMessageHeader(
             when (this@describeFailure) {
-                is CompilerFailure -> appendLine("Compilation failed.")
-                is UnexpectedFailure -> appendLine("Compilation failed with unexpected exception.")
+                is CompilerFailure -> "Compilation failed."
+                is UnexpectedFailure -> "Compilation failed with unexpected exception."
             }
-            appendLine()
-            appendLine(loggedData)
-        }
+        )
 
         private fun DependencyFailures.describeDependencyFailures() =
             buildString {
@@ -270,13 +270,13 @@ private class TestCompilationImpl(
         val loggedCompilerParameters = LoggedData.CompilerParameters(compilerArgs, sourceModules)
 
         val (loggedCompilerCall: LoggedData, result: TestCompilationResult.ImmediateResult) = try {
-            val (exitCode, compilerOutput, compilerOutputHasErrors, durationMillis) = callCompiler(
+            val (exitCode, compilerOutput, compilerOutputHasErrors, duration) = callCompiler(
                 compilerArgs = compilerArgs,
                 lazyKotlinNativeClassLoader = environment.globalEnvironment.lazyKotlinNativeClassLoader
             )
 
             val loggedCompilerCall =
-                LoggedData.CompilerCall(loggedCompilerParameters, exitCode, compilerOutput, compilerOutputHasErrors, durationMillis)
+                LoggedData.CompilerCall(loggedCompilerParameters, exitCode, compilerOutput, compilerOutputHasErrors, duration)
 
             val result = if (exitCode != ExitCode.OK || compilerOutputHasErrors)
                 TestCompilationResult.CompilerFailure(loggedCompilerCall)
@@ -331,7 +331,8 @@ private fun callCompiler(compilerArgs: Array<String>, lazyKotlinNativeClassLoade
     val compilerXmlOutput: ByteArrayOutputStream
     val exitCode: ExitCode
 
-    val durationMillis = measureTimeMillis {
+    @OptIn(ExperimentalTime::class)
+    val duration = measureTime {
         val kotlinNativeClassLoader by lazyKotlinNativeClassLoader
 
         val servicesClass = Class.forName(Services::class.java.canonicalName, true, kotlinNativeClassLoader)
@@ -372,14 +373,14 @@ private fun callCompiler(compilerArgs: Array<String>, lazyKotlinNativeClassLoade
         compilerOutput = outputStream.toString(Charsets.UTF_8.name())
     }
 
-    return CompilerCallResult(exitCode, compilerOutput, messageCollector.hasErrors(), durationMillis)
+    return CompilerCallResult(exitCode, compilerOutput, messageCollector.hasErrors(), duration)
 }
 
 private data class CompilerCallResult(
     val exitCode: ExitCode,
     val compilerOutput: String,
     val compilerOutputHasErrors: Boolean,
-    val durationMillis: Long
+    val duration: Duration
 )
 
 private fun getLogFile(expectedArtifactFile: File): File = expectedArtifactFile.resolveSibling(expectedArtifactFile.name + ".log")

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestEnvironment.kt

@@ -11,6 +11,9 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.util.TestDisposable
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
 import java.io.File
 import java.net.URLClassLoader
+import kotlin.time.Duration
+import kotlin.time.Duration.Companion.seconds
+import kotlin.time.Duration.Companion.milliseconds
 
 internal class TestEnvironment(
     val globalEnvironment: GlobalTestEnvironment,
@@ -27,6 +30,7 @@ internal class GlobalTestEnvironment(
     val lazyKotlinNativeClassLoader: Lazy<ClassLoader> = defaultKotlinNativeClassLoader,
     val testMode: TestMode = defaultTestMode,
     val cacheSettings: TestCacheSettings = defaultCacheSettings,
+    val executionTimeout: Duration = defaultExecutionTimeout,
     val baseBuildDir: File = projectBuildDir
 ) {
     val hostTarget: KonanTarget = HostManager.host
@@ -73,6 +77,15 @@ internal class GlobalTestEnvironment(
             if (useCache) TestCacheSettings.WithCache else TestCacheSettings.WithoutCache
         }
 
+        private val defaultExecutionTimeout: Duration = run {
+            val executionTimeoutValue = System.getProperty(EXECUTION_TIMEOUT)
+            if (executionTimeoutValue != null) {
+                executionTimeoutValue.toLongOrNull()?.milliseconds
+                    ?: fail { "Invalid value for $EXECUTION_TIMEOUT system property: $executionTimeoutValue" }
+            } else
+                DEFAULT_EXECUTION_TIMEOUT
+        }
+
         private val projectBuildDir: File
             get() = System.getenv(PROJECT_BUILD_DIR)?.let(::File) ?: fail { "Non-specified $PROJECT_BUILD_DIR environment variable" }
 
@@ -80,7 +93,10 @@ internal class GlobalTestEnvironment(
         private const val COMPILER_CLASSPATH = "kotlin.internal.native.test.compilerClasspath"
         private const val TEST_MODE = "kotlin.internal.native.test.mode"
         private const val USE_CACHE = "kotlin.internal.native.test.useCache"
+        private const val EXECUTION_TIMEOUT = "kotlin.internal.native.test.executionTimeout"
         private const val PROJECT_BUILD_DIR = "PROJECT_BUILD_DIR"
+
+        private val DEFAULT_EXECUTION_TIMEOUT get() = 10.seconds // Use no backing field to avoid null-initialized value.
     }
 }
 

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestRunProvider.kt

@@ -76,7 +76,7 @@ internal class TestRunProvider(
 
     // Currently, only local test runner is supported.
     fun createRunner(testRun: TestRun): AbstractRunner<*> = when (val target = environment.globalEnvironment.target) {
-        environment.globalEnvironment.hostTarget -> LocalTestRunner(testRun)
+        environment.globalEnvironment.hostTarget -> LocalTestRunner(testRun, environment.globalEnvironment.executionTimeout)
         else -> fail {
             """
                 Running at non-host target is not supported yet.

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractLocalProcessRunner.kt

@@ -6,32 +6,47 @@
 package org.jetbrains.kotlin.konan.blackboxtest.support.runner
 
 import org.jetbrains.kotlin.konan.blackboxtest.support.TestExecutable
+import org.jetbrains.kotlin.konan.blackboxtest.support.runner.AbstractRunner.AbstractRun
+import kotlin.time.*
 
-internal abstract class AbstractLocalProcessRunner<R> : AbstractRunner<R>() {
+internal abstract class AbstractLocalProcessRunner<R>(private val executionTimeout: Duration) : AbstractRunner<R>() {
     protected abstract val visibleProcessName: String
     protected abstract val executable: TestExecutable
     protected abstract val programArgs: List<String>
 
     protected open fun customizeProcess(process: Process) = Unit
 
+    @OptIn(ExperimentalTime::class)
     final override fun buildRun() = AbstractRun {
-        val startTimeMillis = System.currentTimeMillis()
+        val exitCode: Int
 
-        val process = ProcessBuilder(programArgs).directory(executable.executableFile.parentFile).start()
-        customizeProcess(process)
+        val stdOut: String
+        val stdErr: String
 
-        val exitCode = process.waitFor()
-        val finishTimeMillis = System.currentTimeMillis()
+        val duration = measureTime {
+            val process = ProcessBuilder(programArgs).directory(executable.executableFile.parentFile).start()
+            customizeProcess(process)
 
-        val stdOut = process.inputStream.bufferedReader().readText()
-        val stdErr = process.errorStream.bufferedReader().readText()
+            val hasFinishedInTime = process.waitFor(
+                executionTimeout.toLong(DurationUnit.MILLISECONDS),
+                DurationUnit.MILLISECONDS.toTimeUnit()
+            )
 
-        RunResult(exitCode, finishTimeMillis - startTimeMillis, stdOut, stdErr)
+            if (!hasFinishedInTime)
+                return@AbstractRun RunResult.TimeoutExceeded(executionTimeout)
+
+            exitCode = process.exitValue()
+
+            stdOut = process.inputStream.bufferedReader().readText()
+            stdErr = process.errorStream.bufferedReader().readText()
+        }
+
+        RunResult.Completed(exitCode, duration, stdOut, stdErr)
     }
 
-    abstract override fun buildResultHandler(runResult: RunResult): ResultHandler // Narrow returned type.
+    abstract override fun buildResultHandler(runResult: RunResult.Completed): ResultHandler // Narrow returned type.
 
-    abstract inner class ResultHandler(runResult: RunResult) : AbstractRunner<R>.ResultHandler(runResult) {
+    abstract inner class ResultHandler(runResult: RunResult.Completed) : AbstractRunner<R>.ResultHandler(runResult) {
         override fun handle(): R {
             verifyExpectation(0, runResult.exitCode) { "$visibleProcessName exited with non-zero code." }
 

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/AbstractRunner.kt

@@ -8,17 +8,26 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner
 import org.jetbrains.kotlin.konan.blackboxtest.support.LoggedData
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertEquals
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue
-import java.lang.AssertionError
+import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
+import kotlin.time.Duration
 
 internal abstract class AbstractRunner<R> {
     protected abstract fun buildRun(): AbstractRun
-    protected abstract fun buildResultHandler(runResult: RunResult): ResultHandler
+    protected abstract fun buildResultHandler(runResult: RunResult.Completed): ResultHandler
+    protected abstract fun getLoggedParameters(): LoggedData.TestRunParameters
     protected abstract fun handleUnexpectedFailure(t: Throwable): Nothing
 
     fun run(): R = try {
         val run = buildRun()
-        val runResult = run.run()
-        val resultHandler = buildResultHandler(runResult)
+
+        val resultHandler = when (val runResult = run.run()) {
+            is RunResult.TimeoutExceeded -> fail {
+                LoggedData.TestRunTimeoutExceeded(getLoggedParameters(), runResult.timeout)
+                    .withErrorMessageHeader("Timeout exceeded during test execution.")
+            }
+            is RunResult.Completed -> buildResultHandler(runResult)
+        }
+
         resultHandler.handle()
     } catch (t: Throwable) {
         if (t is AssertionError)
@@ -33,29 +42,21 @@ internal abstract class AbstractRunner<R> {
         fun run(): RunResult
     }
 
-    data class RunResult(val exitCode: Int, val durationMillis: Long, val stdOut: String, val stdErr: String)
+    sealed interface RunResult {
+        data class Completed(val exitCode: Int, val duration: Duration, val stdOut: String, val stdErr: String) : RunResult
+        data class TimeoutExceeded(val timeout: Duration) : RunResult
+    }
 
-    abstract inner class ResultHandler(protected val runResult: RunResult) {
+    abstract inner class ResultHandler(protected val runResult: RunResult.Completed) {
         abstract fun getLoggedRun(): LoggedData
         abstract fun handle(): R
 
-        val exitCode get() = runResult.exitCode
-        val durationMillis get() = runResult.durationMillis
-        val stdOut get() = runResult.stdOut
-        val stdErr get() = runResult.stdErr
-
         protected inline fun <T> verifyExpectation(expected: T, actual: T, crossinline errorMessageHeader: () -> String) {
-            assertEquals(expected, actual) { formatErrorMessage(errorMessageHeader) }
+            assertEquals(expected, actual) { getLoggedRun().withErrorMessageHeader(errorMessageHeader()) }
         }
 
         protected inline fun verifyExpectation(shouldBeTrue: Boolean, crossinline errorMessageHeader: () -> String) {
-            assertTrue(shouldBeTrue) { formatErrorMessage(errorMessageHeader) }
-        }
-
-        private inline fun formatErrorMessage(errorMessageHeader: () -> String) = buildString {
-            appendLine(errorMessageHeader())
-            appendLine()
-            appendLine(getLoggedRun())
+            assertTrue(shouldBeTrue) { getLoggedRun().withErrorMessageHeader(errorMessageHeader()) }
         }
     }
 }

native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/runner/LocalTestRunner.kt

@@ -8,8 +8,12 @@ package org.jetbrains.kotlin.konan.blackboxtest.support.runner
 import com.intellij.openapi.util.text.StringUtilRt.convertLineSeparators
 import org.jetbrains.kotlin.konan.blackboxtest.support.*
 import org.jetbrains.kotlin.test.services.JUnit5Assertions.fail
+import kotlin.time.Duration
 
-internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProcessRunner<Unit>() {
+internal class LocalTestRunner(
+    private val testRun: TestRun,
+    executionTimeout: Duration
+) : AbstractLocalProcessRunner<Unit>(executionTimeout) {
     override val visibleProcessName get() = "Tested process"
     override val executable get() = testRun.executable
 
@@ -18,13 +22,12 @@ internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProc
         testRun.runParameters.forEach { it.applyTo(this) }
     }
 
-    private val loggedParameters: LoggedData.TestRunParameters
-        get() = LoggedData.TestRunParameters(
-            compilerCall = executable.loggedCompilerCall,
-            origin = testRun.origin,
-            runArgs = programArgs,
-            runParameters = testRun.runParameters
-        )
+    override fun getLoggedParameters() = LoggedData.TestRunParameters(
+        compilerCall = executable.loggedCompilerCall,
+        origin = testRun.origin,
+        runArgs = programArgs,
+        runParameters = testRun.runParameters
+    )
 
     override fun customizeProcess(process: Process) {
         testRun.runParameters.get<TestRunParameter.WithInputData> {
@@ -33,18 +36,15 @@ internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProc
         }
     }
 
-    override fun buildResultHandler(runResult: RunResult) = ResultHandler(runResult)
+    override fun buildResultHandler(runResult: RunResult.Completed) = ResultHandler(runResult)
 
     override fun handleUnexpectedFailure(t: Throwable) = fail {
-        buildString {
-            appendLine("Test execution failed with unexpected exception.")
-            appendLine()
-            appendLine(LoggedData.TestRunUnexpectedFailure(loggedParameters, t))
-        }
+        LoggedData.TestRunUnexpectedFailure(getLoggedParameters(), t)
+            .withErrorMessageHeader("Test execution failed with unexpected exception.")
     }
 
-    inner class ResultHandler(runResult: RunResult) : AbstractLocalProcessRunner<Unit>.ResultHandler(runResult) {
-        override fun getLoggedRun() = LoggedData.TestRun(loggedParameters, exitCode, stdOut, stdErr, durationMillis)
+    inner class ResultHandler(runResult: RunResult.Completed) : AbstractLocalProcessRunner<Unit>.ResultHandler(runResult) {
+        override fun getLoggedRun() = LoggedData.TestRun(getLoggedParameters(), runResult)
 
         override fun doHandle() {
             if (testRun.runParameters.has<TestRunParameter.WithGTestLogger>()) {
@@ -59,7 +59,7 @@ internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProc
             val cleanStdOut = StringBuilder()
 
             var expectStatusLine = false
-            stdOut.lines().forEach { line ->
+            runResult.stdOut.lines().forEach { line ->
                 when {
                     expectStatusLine -> {
                         val matcher = GTEST_STATUS_LINE_REGEX.matchEntire(line)
@@ -94,10 +94,10 @@ internal class LocalTestRunner(private val testRun: TestRun) : AbstractLocalProc
             val failedTests = (testStatuses - GTEST_STATUS_OK).values.sumOf { it.size }
             verifyExpectation(0, failedTests) { "There are failed tests." }
 
-            verifyOutputData(mergedOutput = cleanStdOut.toString() + stdErr)
+            verifyOutputData(mergedOutput = cleanStdOut.toString() + runResult.stdErr)
         }
 
-        private fun verifyPlainTest() = verifyOutputData(mergedOutput = stdOut + stdErr)
+        private fun verifyPlainTest() = verifyOutputData(mergedOutput = runResult.stdOut + runResult.stdErr)
 
         private fun verifyOutputData(mergedOutput: String) {
             testRun.runParameters.get<TestRunParameter.WithExpectedOutputData> {