From 5ddc31e932f8cfd2b59a1767bbfde7c96a583c4f Mon Sep 17 00:00:00 2001 From: Pavel Kunyavskiy Date: Mon, 5 Feb 2024 16:00:23 +0100 Subject: [PATCH] [Tests] Add handler to regenerate irText testData It is disabled by default, can be enabled by code change. It's useful when changes in dumper is done with significant amount of tests changed. ^KT-65460 --- compiler/test-infrastructure/ReadMe.md | 11 +++++ .../backend/handlers/IrTextDumpHandler.kt | 6 +-- .../backend/handlers/UpdateTestDataHandler.kt | 45 +++++++++++++++++++ .../runners/AbstractKotlinCompilerTest.kt | 3 ++ .../buildsrc-compat/src/main/kotlin/tasks.kt | 1 + 5 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/UpdateTestDataHandler.kt diff --git a/compiler/test-infrastructure/ReadMe.md b/compiler/test-infrastructure/ReadMe.md index 12edc267042..5c764f5e7ed 100644 --- a/compiler/test-infrastructure/ReadMe.md +++ b/compiler/test-infrastructure/ReadMe.md @@ -362,6 +362,17 @@ Values of the debug mode: `0` (or `false`), `1` (or `true`), `2`. Debug mode `2` will ensure that IR is dumped to a file after each lowering phase. The IR dumps will appear next to the generated `.js` or `.wat` file. +# Massive testdata updating + +There is a handler [UpdateTestDataHandler](../tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/UpdateTestDataHandler.kt), +which can be used to update all testData. It is disabled by default. It can be enabled by either changing code, +or by passing system property `kotlin.test.update.test.data`. + +For example, to update all IR text test data by the output of the JVM backend, you can use: +```bash +./gradlew -Pkotlin.test.update.test.data=true :compiler:fir:fir2ir:test --tests "org.jetbrains.kotlin.test.runners.ir.FirPsiJvmIrTextTestGenerated" --continue +``` + # Code style Please keep your abstract test runners as simple as possible. Ideally each abstract test runner should contain **only** test configuration with DSL and nothing else. All services implementations should be declared in separate files. diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/IrTextDumpHandler.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/IrTextDumpHandler.kt index e45f414ae05..4a4846f4154 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/IrTextDumpHandler.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/IrTextDumpHandler.kt @@ -27,10 +27,7 @@ import org.jetbrains.kotlin.test.directives.CodegenTestDirectives.EXTERNAL_FILE import org.jetbrains.kotlin.test.directives.FirDiagnosticsDirectives import org.jetbrains.kotlin.test.directives.FirDiagnosticsDirectives.FIR_IDENTICAL import org.jetbrains.kotlin.test.directives.model.DirectivesContainer -import org.jetbrains.kotlin.test.model.BackendKind -import org.jetbrains.kotlin.test.model.FrontendKinds -import org.jetbrains.kotlin.test.model.TestFile -import org.jetbrains.kotlin.test.model.TestModule +import org.jetbrains.kotlin.test.model.* import org.jetbrains.kotlin.test.services.TestServices import org.jetbrains.kotlin.test.services.moduleStructure import org.jetbrains.kotlin.test.utils.MultiModuleInfoDumper @@ -154,3 +151,4 @@ class IrTextDumpHandler( return computeDumpExtension(this, DUMP_EXTENSION, ignoreFirIdentical) } } + diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/UpdateTestDataHandler.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/UpdateTestDataHandler.kt new file mode 100644 index 00000000000..6f543e6fc94 --- /dev/null +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/backend/handlers/UpdateTestDataHandler.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2010-2024 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.test.backend.handlers + +import com.intellij.rt.execution.junit.FileComparisonFailure +import org.jetbrains.kotlin.test.WrappedException +import org.jetbrains.kotlin.test.model.AfterAnalysisChecker +import org.jetbrains.kotlin.test.services.TestServices +import java.io.File +import kotlin.collections.mapNotNull +import kotlin.io.writeText +import kotlin.text.endsWith + +/** + * Does nothing normally. + * + * If change [enabled] to true or set system property `kotlin.test.update.test.data` to true, + * then all text dump related files would be rewritten on test failures. + * + * This is useful to update test data after change of dump format. + * + * No rewrite happens on muted tests. + * + * For example, if you want to update IR text test data by K2 with the JVM backend, you need to run: + * + * ./gradlew :compiler:fir:fir2ir:test --tests "org.jetbrains.kotlin.test.runners.ir.FirPsiJvmIrTextTestGenerated" --continue -Pkotlin.test.update.test.data=true + * + */ +class UpdateTestDataHandler( + testServices: TestServices +) : AfterAnalysisChecker(testServices) { + private val enabled = false + + override fun suppressIfNeeded(failedAssertions: List): List { + if (enabled || System.getProperty("kotlin.test.update.test.data") == "true") { + for (failure in failedAssertions.mapNotNull { it.cause as? FileComparisonFailure }) { + File(failure.filePath).writeText(failure.actual) + } + } + return failedAssertions + } +} \ No newline at end of file diff --git a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/runners/AbstractKotlinCompilerTest.kt b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/runners/AbstractKotlinCompilerTest.kt index 28815309793..3ef069f931c 100644 --- a/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/runners/AbstractKotlinCompilerTest.kt +++ b/compiler/tests-common-new/tests/org/jetbrains/kotlin/test/runners/AbstractKotlinCompilerTest.kt @@ -8,6 +8,7 @@ package org.jetbrains.kotlin.test.runners import com.intellij.testFramework.TestDataFile import org.jetbrains.kotlin.test.Constructor import org.jetbrains.kotlin.test.ExecutionListenerBasedDisposableProvider +import org.jetbrains.kotlin.test.backend.handlers.UpdateTestDataHandler import org.jetbrains.kotlin.test.builders.TestConfigurationBuilder import org.jetbrains.kotlin.test.builders.testRunner import org.jetbrains.kotlin.test.directives.ConfigurationDirectives @@ -55,6 +56,8 @@ abstract class AbstractKotlinCompilerTest { useAdditionalService { createApplicationDisposableProvider() } useAdditionalService { createKotlinStandardLibrariesPathProvider() } configure(this) + // UpdateTestDataHandler should be the last handler, so it's added after the configure call. + useAfterAnalysisCheckers(::UpdateTestDataHandler) } abstract fun TestConfigurationBuilder.configuration() diff --git a/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/tasks.kt b/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/tasks.kt index 04d490e846c..df8f135f90b 100644 --- a/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/tasks.kt +++ b/repo/gradle-build-conventions/buildsrc-compat/src/main/kotlin/tasks.kt @@ -221,6 +221,7 @@ fun Project.projectTest( environment("PROJECT_BUILD_DIR", project.layout.buildDirectory.get().asFile) systemProperty("jps.kotlin.home", project.rootProject.extra["distKotlinHomeDir"]!!) systemProperty("org.jetbrains.kotlin.skip.muted.tests", if (project.rootProject.hasProperty("skipMutedTests")) "true" else "false") + systemProperty("kotlin.test.update.test.data", if (project.rootProject.hasProperty("kotlin.test.update.test.data")) "true" else "false") systemProperty("cacheRedirectorEnabled", project.rootProject.findProperty("cacheRedirectorEnabled")?.toString() ?: "false") project.kotlinBuildProperties.junit5NumberOfThreadsForParallelExecution?.let { n -> systemProperty("junit.jupiter.execution.parallel.config.strategy", "fixed")