From 9ba5bf72ce00be7a6540cc53c609e97efbe670f5 Mon Sep 17 00:00:00 2001 From: Dmitriy Dolovov Date: Tue, 15 Feb 2022 17:46:02 +0300 Subject: [PATCH] [Native][tests] Support one/two stage test modes --- .../blackboxtest/support/NativeTestSupport.kt | 2 +- .../konan/blackboxtest/support/TestCase.kt | 4 - .../compilation/TestCompilationFactory.kt | 87 ++++++++++++++----- .../support/settings/TestProcessSettings.kt | 20 +++-- 4 files changed, 77 insertions(+), 36 deletions(-) diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt index 8a446bcecfd..7ecf63e499b 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/NativeTestSupport.kt @@ -212,7 +212,7 @@ private object NativeTestSupport { } private fun computeTestMode(enforcedProperties: EnforcedProperties): TestMode = - ClassLevelProperty.TEST_MODE.readValue(enforcedProperties, TestMode.values(), default = TestMode.WITH_MODULES) + ClassLevelProperty.TEST_MODE.readValue(enforcedProperties, TestMode.values(), default = TestMode.TWO_STAGE_MULTI_MODULE) private fun computeTimeouts(enforcedProperties: EnforcedProperties): Timeouts { val executionTimeout = ClassLevelProperty.EXECUTION_TIMEOUT.readValue( diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt index d42ac29a060..479ce81f3f5 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/TestCase.kt @@ -9,7 +9,6 @@ package org.jetbrains.kotlin.konan.blackboxtest.support import org.jetbrains.kotlin.konan.blackboxtest.support.TestCase.WithTestRunnerExtras import org.jetbrains.kotlin.konan.blackboxtest.support.TestModule.Companion.allDependencies -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.TestMode import org.jetbrains.kotlin.konan.blackboxtest.support.util.* import org.jetbrains.kotlin.storage.LockBasedStorageManager import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue @@ -74,9 +73,6 @@ internal class TestFile private constructor( * Represents a module in terms of Kotlin compiler. Includes one or more [TestFile]s. Can be compiled to executable file, KLIB * or any other artifact supported by the Kotlin/Native compiler. * - * Please note that in certain test modes (ex: [TestMode.ONE_STAGE], [TestMode.TWO_STAGE]) modules represented by [TestModule] entities - * are ignored, and all [TestFile]s are compiled together with just the single compiler invocation. - * * [TestModule.Exclusive] represents a collection of [TestFile]s used exclusively for an individual [TestCase]. * [TestModule.Shared] represents a "shared" module, i.e. the auxiliary module that can be used in multiple [TestCase]s. * Such module is compiled to KLIB diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/compilation/TestCompilationFactory.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/compilation/TestCompilationFactory.kt index 5be6001f3f3..3479c810019 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/compilation/TestCompilationFactory.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/compilation/TestCompilationFactory.kt @@ -13,11 +13,9 @@ import org.jetbrains.kotlin.konan.blackboxtest.support.TestModule.Companion.allD import org.jetbrains.kotlin.konan.blackboxtest.support.TestModule.Companion.allFriends import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationArtifact.* import org.jetbrains.kotlin.konan.blackboxtest.support.compilation.TestCompilationDependencyType.* -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Binaries -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.CacheMode -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.KotlinNativeTargets -import org.jetbrains.kotlin.konan.blackboxtest.support.settings.Settings +import org.jetbrains.kotlin.konan.blackboxtest.support.settings.* import org.jetbrains.kotlin.konan.blackboxtest.support.util.* +import org.jetbrains.kotlin.test.services.JUnit5Assertions.assertTrue import org.jetbrains.kotlin.utils.addIfNotNull import java.io.File @@ -32,13 +30,26 @@ internal class TestCompilationFactory { private data class KlibCompilations(val klib: TestCompilation, val staticCache: TestCompilation?) private data class SourceBasedCompilationDependencies( - val klibDependencies: List>, - val staticCacheDependencies: List> + private val klibDependencies: List>, + private val staticCacheDependencies: List> ) { - fun all(): Iterable> = (klibDependencies.asSequence() + staticCacheDependencies).asIterable() + /** Dependencies needed to compile KLIB. */ + fun forKlib(): Iterable> = klibDependencies - fun staticCacheDependenciesWith(klib: CompiledDependency): Iterable> = + /** Dependencies needed to compile KLIB static cache. */ + fun forStaticCache(klib: CompiledDependency): Iterable> = (staticCacheDependencies.asSequence() + klib).asIterable() + + /** Dependencies needed to compile one-stage executable. */ + fun forOneStageExecutable(): Iterable> = + (klibDependencies.asSequence() + staticCacheDependencies).asIterable() + + /** Dependencies needed to compile two-stage executable. */ + fun forTwoStageExecutable( + includedKlib: CompiledDependency, + includedKlibStaticCache: CompiledDependency? + ): Iterable> = + (klibDependencies.asSequence() + staticCacheDependencies + listOfNotNull(includedKlib, includedKlibStaticCache)).asIterable() } fun testCasesToExecutable(testCases: Collection, settings: Settings): TestCompilation { @@ -51,23 +62,44 @@ internal class TestCompilationFactory { // Long pass. val freeCompilerArgs = rootModules.first().testCase.freeCompilerArgs // Should be identical inside the same test case group. val extras = testCases.first().extras // Should be identical inside the same test case group. - val dependencies = collectDependencies(rootModules, freeCompilerArgs, settings).all() val executableArtifact = Executable(settings.artifactFileForExecutable(rootModules)) + val sourceModulesToCompileExecutable: Set + val dependenciesToCompileExecutable: Iterable> + + when (settings.get()) { + TestMode.ONE_STAGE_MULTI_MODULE -> { + // Collect dependencies of root modules. Compile root modules directly to executable. + sourceModulesToCompileExecutable = rootModules + dependenciesToCompileExecutable = collectDependencies(rootModules, freeCompilerArgs, settings).forOneStageExecutable() + } + TestMode.TWO_STAGE_MULTI_MODULE -> { + // Compile root modules to KLIB. Pass this KLIB as included dependency to executable compilation. + val klibCompilations = modulesToKlib(rootModules, freeCompilerArgs, settings) + + sourceModulesToCompileExecutable = emptySet() // No sources. + + // Include just compiled KLIB as -Xinclude dependency. + dependenciesToCompileExecutable = collectDependencies(rootModules, freeCompilerArgs, settings).forTwoStageExecutable( + includedKlib = klibCompilations.klib.asKlibDependency(IncludedLibrary), + includedKlibStaticCache = klibCompilations.staticCache?.asStaticCacheDependency() + ) + } + } + return cachedExecutableCompilations.computeIfAbsent(cacheKey) { ExecutableCompilation( settings = settings, freeCompilerArgs = freeCompilerArgs, - sourceModules = rootModules, + sourceModules = sourceModulesToCompileExecutable, extras = extras, - dependencies = dependencies, + dependencies = dependenciesToCompileExecutable, expectedArtifact = executableArtifact ) } } - private fun moduleToKlib(sourceModule: TestModule, freeCompilerArgs: TestCompilerArgs, settings: Settings): KlibCompilations { - val sourceModules = setOf(sourceModule) + private fun modulesToKlib(sourceModules: Set, freeCompilerArgs: TestCompilerArgs, settings: Settings): KlibCompilations { val cacheKey = KlibCacheKey(sourceModules, freeCompilerArgs) // Fast pass. @@ -75,7 +107,7 @@ internal class TestCompilationFactory { // Long pass. val dependencies = collectDependencies(sourceModules, freeCompilerArgs, settings) - val klibArtifact = KLIB(settings.artifactFileForKlib(sourceModule, freeCompilerArgs)) + val klibArtifact = KLIB(settings.artifactFileForKlib(sourceModules, freeCompilerArgs)) val staticCacheArtifact: KLIBStaticCache? = if (settings.get().staticCacheRequiredForEveryLibrary) KLIBStaticCache(cacheDir = klibArtifact.cacheDirForStaticCache(), klib = klibArtifact) @@ -87,7 +119,7 @@ internal class TestCompilationFactory { settings = settings, freeCompilerArgs = freeCompilerArgs, sourceModules = sourceModules, - dependencies = dependencies.klibDependencies, + dependencies = dependencies.forKlib(), expectedArtifact = klibArtifact ) @@ -95,7 +127,7 @@ internal class TestCompilationFactory { StaticCacheCompilation( settings = settings, freeCompilerArgs = freeCompilerArgs, - dependencies = dependencies.staticCacheDependenciesWith(klibCompilation.asKlibDependency(type = /* does not matter in fact*/ Library)), + dependencies = dependencies.forStaticCache(klibCompilation.asKlibDependency(type = /* does not matter in fact*/ Library)), expectedArtifact = staticCacheArtifact ) } else @@ -115,9 +147,11 @@ internal class TestCompilationFactory { fun > Set.collectDependencies(type: T) = forEach { dependencyModule: TestModule -> - val klibCompilations = moduleToKlib(dependencyModule, freeCompilerArgs, settings) + val klibCompilations = modulesToKlib(setOf(dependencyModule), freeCompilerArgs, settings) klibDependencies += klibCompilations.klib.asKlibDependency(type) - staticCacheDependencies.addIfNotNull(klibCompilations.staticCache?.asStaticCacheDependency()) + + if (type == Library || type == IncludedLibrary) + staticCacheDependencies.addIfNotNull(klibCompilations.staticCache?.asStaticCacheDependency()) } sourceModules.allDependencies().collectDependencies(Library) @@ -144,10 +178,19 @@ internal class TestCompilationFactory { private fun Settings.artifactFileForExecutable(module: TestModule.Exclusive) = singleModuleArtifactFile(module, get().testTarget.family.exeSuffix) - private fun Settings.artifactFileForKlib(module: TestModule, freeCompilerArgs: TestCompilerArgs) = when (module) { - is TestModule.Exclusive -> singleModuleArtifactFile(module, "klib") - is TestModule.Shared -> get().sharedBinariesDir.resolve("${module.name}-${prettyHash(freeCompilerArgs.hashCode())}.klib") - } + private fun Settings.artifactFileForKlib(modules: Set, freeCompilerArgs: TestCompilerArgs): File = + when (modules.size) { + 1 -> when (val module = modules.first()) { + is TestModule.Exclusive -> singleModuleArtifactFile(module, "klib") + is TestModule.Shared -> get().sharedBinariesDir.resolve("${module.name}-${prettyHash(freeCompilerArgs.hashCode())}.klib") + } + else -> { + assertTrue(modules.none { module -> module is TestModule.Shared }) { + "Can't compile shared module together with any other module" + } + multiModuleArtifactFile(modules, "klib") + } + } private fun KLIB.cacheDirForStaticCache(): File = klibFile.parentFile.resolve(STATIC_CACHE_DIR_NAME).apply { mkdirs() } diff --git a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt index c7370df535f..2eef713eb8d 100644 --- a/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt +++ b/native/native.tests/tests/org/jetbrains/kotlin/konan/blackboxtest/support/settings/TestProcessSettings.kt @@ -34,17 +34,19 @@ internal class KotlinNativeClassLoader(private val lazyClassLoader: Lazy