DryRun mode for GenerateTests is added

Relates to #KTI-17

compiler/tests-java8/tests/org/jetbrains/kotlin/generators/tests/GenerateJava8Tests.kt

@@ -20,15 +20,16 @@ import org.jetbrains.kotlin.checkers.AbstractForeignJava8AnnotationsNoAnnotation
 import org.jetbrains.kotlin.checkers.AbstractForeignJava8AnnotationsNoAnnotationInClasspathWithPsiClassReadingTest
 import org.jetbrains.kotlin.checkers.AbstractForeignJava8AnnotationsTest
 import org.jetbrains.kotlin.checkers.javac.AbstractJavacForeignJava8AnnotationsTest
-import org.jetbrains.kotlin.generators.tests.generator.testGroup
+import org.jetbrains.kotlin.generators.tests.generator.testGroupSuite
 import org.jetbrains.kotlin.jvm.compiler.AbstractLoadJava8Test
 import org.jetbrains.kotlin.jvm.compiler.AbstractLoadJava8WithPsiClassReadingTest
 import org.jetbrains.kotlin.jvm.compiler.javac.AbstractLoadJava8UsingJavacTest
 import org.jetbrains.kotlin.resolve.calls.AbstractEnhancedSignaturesResolvedCallsTest
 
-fun main() {
+fun main(args: Array<String>) {
     System.setProperty("java.awt.headless", "true")
 
+    testGroupSuite(args) {
         testGroup("compiler/tests-java8/tests", "compiler/testData") {
             testClass<AbstractForeignJava8AnnotationsTest> {
                 model("foreignAnnotationsJava8/tests")
@@ -64,4 +65,5 @@ fun main() {
                 model("resolvedCalls/enhancedSignatures")
             }
         }
+    }
 }

compiler/tests-spec/tests/org/jetbrains/kotlin/spec/utils/tasks/GenerateSpecTests.kt

@@ -5,7 +5,7 @@
 
 package org.jetbrains.kotlin.spec.utils.tasks
 
-import org.jetbrains.kotlin.generators.tests.generator.testGroup
+import org.jetbrains.kotlin.generators.tests.generator.testGroupSuite
 import org.jetbrains.kotlin.spec.checkers.AbstractDiagnosticsTestSpec
 import org.jetbrains.kotlin.spec.checkers.AbstractFirDiagnosticsTestSpec
 import org.jetbrains.kotlin.spec.codegen.AbstractBlackBoxCodegenTestSpec
@@ -38,6 +38,7 @@ fun detectDirsWithTestsMapFileOnly(dirName: String): List<String> {
 fun generateTests() {
     val excludedFirTestdataPattern = "^(.+)\\.fir\\.kts?\$"
 
+    testGroupSuite {
         testGroup(SPEC_TEST_PATH, SPEC_TESTDATA_PATH) {
             testClass<AbstractDiagnosticsTestSpec> {
                 model(
@@ -66,6 +67,7 @@ fun generateTests() {
                 model("codegen/box", excludeDirs = listOf("helpers", "templates") + detectDirsWithTestsMapFileOnly("codegen/box"))
             }
         }
+    }
 }
 
 fun main() {

compiler/tests/org/jetbrains/kotlin/generators/tests/GenerateCompilerTests.kt

@@ -27,7 +27,7 @@ import org.jetbrains.kotlin.fir.java.AbstractFirOldFrontendLightClassesTest
 import org.jetbrains.kotlin.fir.java.AbstractFirTypeEnhancementTest
 import org.jetbrains.kotlin.fir.java.AbstractOwnFirTypeEnhancementTest
 import org.jetbrains.kotlin.fir.lightTree.AbstractLightTree2FirConverterTestCase
-import org.jetbrains.kotlin.generators.tests.generator.testGroup
+import org.jetbrains.kotlin.generators.tests.generator.testGroupSuite
 import org.jetbrains.kotlin.generators.util.KT_OR_KTS_WITHOUT_DOTS_IN_NAME
 import org.jetbrains.kotlin.generators.util.KT_WITHOUT_DOTS_IN_NAME
 import org.jetbrains.kotlin.integration.AbstractAntTaskTest
@@ -59,11 +59,12 @@ import org.jetbrains.kotlin.types.AbstractTypeBindingTest
 import org.jetbrains.kotlin.visualizer.fir.AbstractFirVisualizer
 import org.jetbrains.kotlin.visualizer.psi.AbstractPsiVisualizer
 
-fun main() {
+fun main(args: Array<String>) {
     System.setProperty("java.awt.headless", "true")
 
     val excludedFirTestdataPattern = "^(.+)\\.fir\\.kts?\$"
 
+    testGroupSuite(args) {
         testGroup("compiler/tests", "compiler/testData") {
             testClass<AbstractDiagnosticsTestWithFirValidation>(suiteTestClassName = "DiagnosticsTestGenerated") {
                 model("diagnostics/tests", pattern = "^(.*)\\.kts?$", excludedPattern = excludedFirTestdataPattern)
@@ -176,10 +177,10 @@ fun main() {
                 model("parseCodeFragment/block", testMethod = "doBlockCodeFragmentParsingTest", extension = "kt")
             }
 
-        GenerateRangesCodegenTestData.main(emptyArray())
+            GenerateRangesCodegenTestData.main(args)
-        GenerateInRangeExpressionTestData.main(emptyArray())
+            GenerateInRangeExpressionTestData.main(args)
-        GenerateSteppedRangesCodegenTestData.main(emptyArray())
+            GenerateSteppedRangesCodegenTestData.main(args)
-        GeneratePrimitiveVsObjectEqualityTestData.main(emptyArray())
+            GeneratePrimitiveVsObjectEqualityTestData.main(args)
 
             testClass<AbstractBlackBoxCodegenTest> {
                 model("codegen/box", targetBackend = TargetBackend.JVM)
@@ -264,7 +265,11 @@ fun main() {
             testClass<AbstractLoadJavaTest> {
                 model("loadJava/compiledJava", extension = "java", testMethod = "doTestCompiledJava")
                 model("loadJava/compiledJavaAndKotlin", extension = "txt", testMethod = "doTestCompiledJavaAndKotlin")
-            model("loadJava/compiledJavaIncludeObjectMethods", extension = "java", testMethod = "doTestCompiledJavaIncludeObjectMethods")
+                model(
+                    "loadJava/compiledJavaIncludeObjectMethods",
+                    extension = "java",
+                    testMethod = "doTestCompiledJavaIncludeObjectMethods"
+                )
                 model("loadJava/compiledKotlin", testMethod = "doTestCompiledKotlin")
                 model("loadJava/compiledKotlinWithStdlib", testMethod = "doTestCompiledKotlinWithStdlib")
                 model("loadJava/javaAgainstKotlin", extension = "txt", testMethod = "doTestJavaAgainstKotlin")
@@ -280,7 +285,11 @@ fun main() {
             testClass<AbstractLoadJavaUsingJavacTest> {
                 model("loadJava/compiledJava", extension = "java", testMethod = "doTestCompiledJava")
                 model("loadJava/compiledJavaAndKotlin", extension = "txt", testMethod = "doTestCompiledJavaAndKotlin")
-            model("loadJava/compiledJavaIncludeObjectMethods", extension = "java", testMethod = "doTestCompiledJavaIncludeObjectMethods")
+                model(
+                    "loadJava/compiledJavaIncludeObjectMethods",
+                    extension = "java",
+                    testMethod = "doTestCompiledJavaIncludeObjectMethods"
+                )
                 model("loadJava/compiledKotlin", testMethod = "doTestCompiledKotlin")
                 model("loadJava/compiledKotlinWithStdlib", testMethod = "doTestCompiledKotlinWithStdlib")
                 model("loadJava/javaAgainstKotlin", extension = "txt", testMethod = "doTestJavaAgainstKotlin")
@@ -464,7 +473,11 @@ fun main() {
                     targetBackend = TargetBackend.JVM_IR
                 )
                 model("loadJava/compiledKotlin", testMethod = "doTestCompiledKotlin", targetBackend = TargetBackend.JVM_IR)
-            model("loadJava/compiledKotlinWithStdlib", testMethod = "doTestCompiledKotlinWithStdlib", targetBackend = TargetBackend.JVM_IR)
+                model(
+                    "loadJava/compiledKotlinWithStdlib",
+                    testMethod = "doTestCompiledKotlinWithStdlib",
+                    targetBackend = TargetBackend.JVM_IR
+                )
                 model(
                     "loadJava/javaAgainstKotlin",
                     extension = "txt",
@@ -620,4 +633,5 @@ fun main() {
                 model("uncommonCases/testFiles", testMethod = "doUncommonCasesTest")
             }
         }
+    }
 }

core/descriptors.runtime/tests/org/jetbrains/kotlin/generators/tests/GenerateRuntimeDescriptorTests.kt

@@ -16,13 +16,14 @@
 
 package org.jetbrains.kotlin.generators.tests
 
-import org.jetbrains.kotlin.generators.tests.generator.testGroup
+import org.jetbrains.kotlin.generators.tests.generator.testGroupSuite
 import org.jetbrains.kotlin.jvm.runtime.AbstractJvm8RuntimeDescriptorLoaderTest
 import org.jetbrains.kotlin.jvm.runtime.AbstractJvmRuntimeDescriptorLoaderTest
 
-fun main() {
+fun main(args: Array<String>) {
     System.setProperty("java.awt.headless", "true")
 
+    testGroupSuite(args) {
         testGroup("core/descriptors.runtime/tests", "compiler/testData") {
             testClass<AbstractJvmRuntimeDescriptorLoaderTest> {
                 model("loadJava/compiledKotlin")
@@ -33,4 +34,5 @@ fun main() {
                 model("loadJava8/compiledJava", extension = "java")
             }
         }
+    }
 }

generators/main/GeneratorsFileUtil.kt

@@ -24,7 +24,7 @@ object GeneratorsFileUtil {
                 throw IllegalStateException("Cannot create directory: $parentFile")
             }
         }
-        if (checkFileIgnoringLineSeparators(file, newText)) {
+        if (!isFileContentChangedIgnoringLineSeparators(file, newText)) {
             if (logNotChanged) {
                 println("Not changed: " + file.absolutePath)
             }
@@ -42,12 +42,12 @@ object GeneratorsFileUtil {
         println()
     }
 
-    private fun checkFileIgnoringLineSeparators(file: File, content: String): Boolean {
+    fun isFileContentChangedIgnoringLineSeparators(file: File, content: String): Boolean {
         val currentContent: String = try {
             StringUtil.convertLineSeparators(file.readText(Charsets.UTF_8))
         } catch (ignored: Throwable) {
-            return false
+            return true
         }
-        return StringUtil.convertLineSeparators(content) == currentContent
+        return StringUtil.convertLineSeparators(content) != currentContent
     }
 }

generators/test-generator/tests/org/jetbrains/kotlin/generators/tests/generator/TestGenerationDSL.kt

@@ -6,6 +6,8 @@
 package org.jetbrains.kotlin.generators.tests.generator
 
 import junit.framework.TestCase
+import org.jetbrains.kotlin.generators.tests.generator.InconsistencyChecker.Companion.hasDryRunArg
+import org.jetbrains.kotlin.generators.tests.generator.InconsistencyChecker.Companion.inconsistencyChecker
 import org.jetbrains.kotlin.test.TargetBackend
 import java.io.File
 import java.util.*
@@ -16,7 +18,8 @@ class TestGroup(
     val testDataRoot: String,
     val testRunnerMethodName: String,
     val additionalRunnerArguments: List<String> = emptyList(),
-    val annotations: List<AnnotationModel> = emptyList()
+    val annotations: List<AnnotationModel> = emptyList(),
+    private val dryRun: Boolean = false
 ) {
     inline fun <reified T : TestCase> testClass(
         suiteTestClassName: String = getDefaultSuiteTestClassName(T::class.java.simpleName),
@@ -34,13 +37,16 @@ class TestGroup(
         annotations: List<AnnotationModel> = emptyList(),
         init: TestClass.() -> Unit
     ) {
-        TestGenerator(
+        val testGenerator = TestGenerator(
             testsRoot,
             suiteTestClassName,
             baseTestClassName,
             TestClass(annotations).apply(init).testModels,
             useJunit4
-        ).generateAndSave()
+        )
+        if (testGenerator.generateAndSave(dryRun)) {
+            inconsistencyChecker(dryRun).add(testGenerator.testSourceFilePath)
+        }
     }
 
     inner class TestClass(val annotations: List<AnnotationModel>) {
@@ -85,14 +91,67 @@ class TestGroup(
     }
 }
 
-fun testGroup(
+fun testGroupSuite(
+    args: Array<String>,
+    init: TestGroupSuite.() -> Unit
+) {
+    testGroupSuite(hasDryRunArg(args), init)
+}
+
+fun testGroupSuite(
+    dryRun: Boolean = false,
+    init: TestGroupSuite.() -> Unit
+) {
+    TestGroupSuite(dryRun).init()
+}
+
+class TestGroupSuite(private val dryRun: Boolean) {
+    fun testGroup(
         testsRoot: String,
         testDataRoot: String,
         testRunnerMethodName: String = RunTestMethodModel.METHOD_NAME,
         additionalRunnerArguments: List<String> = emptyList(),
         init: TestGroup.() -> Unit
-) {
+    ) {
-    TestGroup(testsRoot, testDataRoot, testRunnerMethodName, additionalRunnerArguments).init()
+        TestGroup(
+            testsRoot,
+            testDataRoot,
+            testRunnerMethodName,
+            additionalRunnerArguments,
+            dryRun = dryRun
+        ).init()
+    }
+}
+
+interface InconsistencyChecker {
+    fun add(affectedFile: String)
+
+    val affectedFiles: List<String>
+
+    companion object {
+        fun hasDryRunArg(args: Array<String>) = args.any { it == "dryRun" }
+
+        fun inconsistencyChecker(dryRun: Boolean) = if (dryRun) DefaultInconsistencyChecker else EmptyInconsistencyChecker
+    }
+}
+
+object DefaultInconsistencyChecker : InconsistencyChecker {
+    private val files = mutableListOf<String>()
+
+    override fun add(affectedFile: String) {
+        files.add(affectedFile)
+    }
+
+    override val affectedFiles: List<String>
+        get() = files
+}
+
+object EmptyInconsistencyChecker : InconsistencyChecker {
+    override fun add(affectedFile: String) {
+    }
+
+    override val affectedFiles: List<String>
+        get() = emptyList()
 }
 
 fun getDefaultSuiteTestClassName(baseTestClassName: String): String {

generators/test-generator/tests/org/jetbrains/kotlin/generators/tests/generator/TestGenerator.kt

@@ -8,7 +8,6 @@ package org.jetbrains.kotlin.generators.tests.generator
 import org.jetbrains.kotlin.generators.util.GeneratorsFileUtil
 import org.jetbrains.kotlin.test.JUnit3RunnerWithInners
 import org.jetbrains.kotlin.test.KotlinTestUtils
-import org.jetbrains.kotlin.test.TargetBackend
 import org.jetbrains.kotlin.test.TestMetadata
 import org.jetbrains.kotlin.utils.Printer
 import org.junit.Test
@@ -31,7 +30,7 @@ class TestGenerator(
     private val baseTestClassName: String
     private val testClassModels: Collection<TestClassModel>
     private val useJunit4: Boolean
-    private val testSourceFilePath: String
+    internal val testSourceFilePath: String
 
     init {
         this.baseTestClassPackage = baseTestClassFqName.substringBeforeLast('.', "")
@@ -48,8 +47,23 @@ class TestGenerator(
         }
     }
 
+    /**
+     * @return true if a new file is generated
+     */
     @Throws(IOException::class)
-    fun generateAndSave() {
+    fun generateAndSave(dryRun: Boolean): Boolean {
+        val generatedCode = generate()
+
+        val testSourceFile = File(testSourceFilePath)
+        val changed =
+            GeneratorsFileUtil.isFileContentChangedIgnoringLineSeparators(testSourceFile, generatedCode)
+        if (!dryRun) {
+            GeneratorsFileUtil.writeFileIfContentChanged(testSourceFile, generatedCode, false)
+        }
+        return changed
+    }
+
+    private fun generate(): String {
         val out = StringBuilder()
         val p = Printer(out)
 
@@ -126,9 +140,7 @@ class TestGenerator(
         }
 
         generateTestClass(p, model, false)
-
+        return out.toString()
-        val testSourceFile = File(testSourceFilePath)
-        GeneratorsFileUtil.writeFileIfContentChanged(testSourceFile, out.toString(), false)
     }
 
     private fun generateTestClass(p: Printer, testClassModel: TestClassModel, isStatic: Boolean) {

generators/tests/org/jetbrains/kotlin/generators/tests/GenerateTests.kt

@@ -31,7 +31,7 @@ import org.jetbrains.kotlin.formatter.AbstractFormatterTest
 import org.jetbrains.kotlin.formatter.AbstractTypingIndentationTestBase
 import org.jetbrains.kotlin.generators.tests.generator.TestGroup
 import org.jetbrains.kotlin.generators.tests.generator.muteExtraSuffix
-import org.jetbrains.kotlin.generators.tests.generator.testGroup
+import org.jetbrains.kotlin.generators.tests.generator.testGroupSuite
 import org.jetbrains.kotlin.generators.util.KT_OR_KTS
 import org.jetbrains.kotlin.generators.util.KT_OR_KTS_WITHOUT_DOTS_IN_NAME
 import org.jetbrains.kotlin.generators.util.KT_WITHOUT_DOTS_IN_NAME
@@ -181,9 +181,10 @@ import org.jetbrains.kotlinx.serialization.AbstractSerializationIrBytecodeListin
 import org.jetbrains.kotlinx.serialization.AbstractSerializationPluginBytecodeListingTest
 import org.jetbrains.kotlinx.serialization.AbstractSerializationPluginDiagnosticTest
 
-fun main() {
+fun main(args: Array<String>) {
     System.setProperty("java.awt.headless", "true")
 
+    testGroupSuite(args) {
         testGroup("idea/jvm-debugger/jvm-debugger-test/test", "idea/jvm-debugger/jvm-debugger-test/testData") {
             testClass<AbstractKotlinSteppingTest> {
                 model(
@@ -525,7 +526,10 @@ fun main() {
                 model("refactoring/safeDelete/deleteTypeParameter/kotlinTypeParameter", testMethod = "doTypeParameterTest")
                 model("refactoring/safeDelete/deleteTypeParameter/kotlinTypeParameterWithJava", testMethod = "doTypeParameterTestWithJava")
                 model("refactoring/safeDelete/deleteValueParameter/kotlinValueParameter", testMethod = "doValueParameterTest")
-            model("refactoring/safeDelete/deleteValueParameter/kotlinValueParameterWithJava", testMethod = "doValueParameterTestWithJava")
+                model(
+                    "refactoring/safeDelete/deleteValueParameter/kotlinValueParameterWithJava",
+                    testMethod = "doValueParameterTestWithJava"
+                )
             }
 
             testClass<AbstractReferenceResolveTest> {
@@ -963,7 +967,13 @@ fun main() {
                     testClassName = "WorksheetCompiling",
                     recursive = false
                 )
-            model("worksheet", extension = "ws.kts", testMethod = "doWorksheetReplTest", testClassName = "WorksheetRepl", recursive = false)
+                model(
+                    "worksheet",
+                    extension = "ws.kts",
+                    testMethod = "doWorksheetReplTest",
+                    testClassName = "WorksheetRepl",
+                    recursive = false
+                )
                 model(
                     "worksheet/multiFile",
                     extension = null,
@@ -1587,4 +1597,5 @@ fun main() {
         }
     }
 */
+    }
 }

generators/tests/org/jetbrains/kotlin/generators/tests/GenerateTests.kt.as36

@@ -31,7 +31,7 @@ import org.jetbrains.kotlin.formatter.AbstractFormatterTest
 import org.jetbrains.kotlin.formatter.AbstractTypingIndentationTestBase
 import org.jetbrains.kotlin.generators.tests.generator.TestGroup
 import org.jetbrains.kotlin.generators.tests.generator.muteExtraSuffix
-import org.jetbrains.kotlin.generators.tests.generator.testGroup
+import org.jetbrains.kotlin.generators.tests.generator.testGroupSuite
 import org.jetbrains.kotlin.generators.util.KT_OR_KTS
 import org.jetbrains.kotlin.generators.util.KT_OR_KTS_WITHOUT_DOTS_IN_NAME
 import org.jetbrains.kotlin.generators.util.KT_WITHOUT_DOTS_IN_NAME
@@ -162,9 +162,10 @@ import org.jetbrains.kotlinx.serialization.AbstractSerializationPluginBytecodeLi
 import org.jetbrains.kotlinx.serialization.AbstractSerializationPluginDiagnosticTest
 import org.jetbrains.kotlinx.serialization.AbstractSerializationIrBytecodeListingTest
 
-fun main() {
+fun main(args: Array<String>) {
     System.setProperty("java.awt.headless", "true")
 
+    testGroupSuite(args) {
         testGroup("idea/jvm-debugger/jvm-debugger-test/test", "idea/jvm-debugger/jvm-debugger-test/testData") {
             testClass<AbstractKotlinSteppingTest> {
                 model(
@@ -1037,9 +1038,9 @@ fun main() {
             }
         }
 
-/*  There is no jps in AS
+    /*  There is no jps in AS
         ....
-*/
+    */
         testGroup("compiler/incremental-compilation-impl/test", "jps-plugin/testData") {
             testClass<AbstractIncrementalJvmCompilerRunnerTest> {
                 model("incremental/pureKotlin", extension = null, recursive = false)
@@ -1147,7 +1148,7 @@ fun main() {
                 model("codegen")
             }
         }
-/*
+    /*
         testGroup("plugins/android-extensions/android-extensions-idea/tests", "plugins/android-extensions/android-extensions-idea/testData") {
             testClass<AbstractAndroidCompletionTest> {
                 model("android/completion", recursive = false, extension = null)
@@ -1220,5 +1221,6 @@ fun main() {
                 model("android/gutterIcon")
             }
         }
-*/
+    */
+    }
 }

generators/tests/org/jetbrains/kotlin/generators/tests/GenerateTests.kt.as40

@@ -31,7 +31,7 @@ import org.jetbrains.kotlin.formatter.AbstractFormatterTest
 import org.jetbrains.kotlin.formatter.AbstractTypingIndentationTestBase
 import org.jetbrains.kotlin.generators.tests.generator.TestGroup
 import org.jetbrains.kotlin.generators.tests.generator.muteExtraSuffix
-import org.jetbrains.kotlin.generators.tests.generator.testGroup
+import org.jetbrains.kotlin.generators.tests.generator.testGroupSuite
 import org.jetbrains.kotlin.generators.util.KT_OR_KTS
 import org.jetbrains.kotlin.generators.util.KT_OR_KTS_WITHOUT_DOTS_IN_NAME
 import org.jetbrains.kotlin.generators.util.KT_WITHOUT_DOTS_IN_NAME
@@ -162,9 +162,10 @@ import org.jetbrains.kotlinx.serialization.AbstractSerializationPluginBytecodeLi
 import org.jetbrains.kotlinx.serialization.AbstractSerializationPluginDiagnosticTest
 import org.jetbrains.kotlinx.serialization.AbstractSerializationIrBytecodeListingTest
 
-fun main() {
+fun main(args: Array<String>) {
     System.setProperty("java.awt.headless", "true")
 
+    testGroupSuite(args) {
         testGroup("idea/jvm-debugger/jvm-debugger-test/test", "idea/jvm-debugger/jvm-debugger-test/testData") {
             testClass<AbstractKotlinSteppingTest> {
                 model(
@@ -1036,9 +1037,9 @@ fun main() {
             }
         }
 
-/*  There is no jps in AS
+    /*  There is no jps in AS
         ....
-*/
+    */
         testGroup("compiler/incremental-compilation-impl/test", "jps-plugin/testData") {
             testClass<AbstractIncrementalJvmCompilerRunnerTest> {
                 model("incremental/pureKotlin", extension = null, recursive = false)
@@ -1146,7 +1147,7 @@ fun main() {
                 model("codegen")
             }
         }
-/*
+    /*
         testGroup("plugins/android-extensions/android-extensions-idea/tests", "plugins/android-extensions/android-extensions-idea/testData") {
             testClass<AbstractAndroidCompletionTest> {
                 model("android/completion", recursive = false, extension = null)
@@ -1219,5 +1220,6 @@ fun main() {
                 model("android/gutterIcon")
             }
         }
-*/
+    */
+    }
 }

generators/tests/org/jetbrains/kotlin/generators/tests/GenerateTests.kt.as41

@@ -31,7 +31,7 @@ import org.jetbrains.kotlin.formatter.AbstractFormatterTest
 import org.jetbrains.kotlin.formatter.AbstractTypingIndentationTestBase
 import org.jetbrains.kotlin.generators.tests.generator.TestGroup
 import org.jetbrains.kotlin.generators.tests.generator.muteExtraSuffix
-import org.jetbrains.kotlin.generators.tests.generator.testGroup
+import org.jetbrains.kotlin.generators.tests.generator.testGroupSuite
 import org.jetbrains.kotlin.generators.util.KT_OR_KTS
 import org.jetbrains.kotlin.generators.util.KT_OR_KTS_WITHOUT_DOTS_IN_NAME
 import org.jetbrains.kotlin.generators.util.KT_WITHOUT_DOTS_IN_NAME
@@ -165,6 +165,7 @@ import org.jetbrains.kotlinx.serialization.AbstractSerializationIrBytecodeListin
 fun main(args: Array<String>) {
     System.setProperty("java.awt.headless", "true")
 
+    testGroupSuite(args) {
         testGroup("idea/jvm-debugger/jvm-debugger-test/test", "idea/jvm-debugger/jvm-debugger-test/testData") {
             testClass<AbstractKotlinSteppingTest> {
                 model(
@@ -1036,9 +1037,9 @@ fun main(args: Array<String>) {
             }
         }
 
-/*  There is no jps in AS
+    /*  There is no jps in AS
         ....
-*/
+    */
         testGroup("compiler/incremental-compilation-impl/test", "jps-plugin/testData") {
             testClass<AbstractIncrementalJvmCompilerRunnerTest> {
                 model("incremental/pureKotlin", extension = null, recursive = false)
@@ -1146,7 +1147,7 @@ fun main(args: Array<String>) {
                 model("codegen")
             }
         }
-/*
+    /*
         testGroup("plugins/android-extensions/android-extensions-idea/tests", "plugins/android-extensions/android-extensions-idea/testData") {
             testClass<AbstractAndroidCompletionTest> {
                 model("android/completion", recursive = false, extension = null)
@@ -1219,5 +1220,6 @@ fun main(args: Array<String>) {
                 model("android/gutterIcon")
             }
         }
-*/
+    */
+    }
 }

js/js.tests/test/org/jetbrains/kotlin/generators/tests/GenerateJsTests.kt

@@ -5,7 +5,7 @@
 
 package org.jetbrains.kotlin.generators.tests
 
-import org.jetbrains.kotlin.generators.tests.generator.testGroup
+import org.jetbrains.kotlin.generators.tests.generator.testGroupSuite
 import org.jetbrains.kotlin.js.test.AbstractDceTest
 import org.jetbrains.kotlin.js.test.AbstractJsLineNumberTest
 import org.jetbrains.kotlin.js.test.es6.semantics.*
@@ -14,12 +14,13 @@ import org.jetbrains.kotlin.js.test.semantics.*
 import org.jetbrains.kotlin.js.test.wasm.semantics.AbstractIrWasmBoxWasmTest
 import org.jetbrains.kotlin.test.TargetBackend
 
-fun main() {
+fun main(args: Array<String>) {
     System.setProperty("java.awt.headless", "true")
 
     // TODO: repair these tests
     //generateTestDataForReservedWords()
 
+    testGroupSuite(args) {
         testGroup("js/js.tests/test", "js/js.translator/testData", testRunnerMethodName = "runTest0") {
             testClass<AbstractBoxJsTest> {
                 model("box/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS)
@@ -100,4 +101,5 @@ fun main() {
                 model("codegen/box/arrays", targetBackend = TargetBackend.JS)
             }
         }
+    }
 }

plugins/pill/generate-all-tests/test/org/jetbrains/kotlin/pill/generateAllTests/Main.java

@@ -6,13 +6,24 @@
 package org.jetbrains.kotlin.pill.generateAllTests;
 
 import org.jetbrains.kotlin.generators.tests.*;
+import org.jetbrains.kotlin.generators.tests.generator.InconsistencyChecker;
+
+import java.util.List;
 
 public class Main {
     public static void main(String[] args) {
-        GenerateCompilerTestsKt.main();
+        GenerateCompilerTestsKt.main(args);
-        GenerateTestsKt.main();
+        GenerateTestsKt.main(args);
-        GenerateJsTestsKt.main();
+        GenerateJsTestsKt.main(args);
-        GenerateJava8TestsKt.main();
+        GenerateJava8TestsKt.main(args);
-        GenerateRuntimeDescriptorTestsKt.main();
+        GenerateRuntimeDescriptorTestsKt.main(args);
+
+        boolean dryRun = InconsistencyChecker.Companion.hasDryRunArg(args);
+        List<String> affectedFiles = InconsistencyChecker.Companion.inconsistencyChecker(dryRun).getAffectedFiles();
+        int size = affectedFiles.size();
+        if (size > 0) {
+            throw new IllegalStateException("There " + (size == 1 ? "is a test" : "are " + size + " tests") + " to be regenerated:\n"
+                                            + String.join("\n", affectedFiles));
+        }
     }
 }