DryRun mode for GenerateTests is added

Relates to #KTI-17

compiler/tests-java8/tests/org/jetbrains/kotlin/generators/tests/GenerateJava8Tests.kt

@@ -20,48 +20,50 @@ import org.jetbrains.kotlin.checkers.AbstractForeignJava8AnnotationsNoAnnotation
 import org.jetbrains.kotlin.checkers.AbstractForeignJava8AnnotationsNoAnnotationInClasspathWithPsiClassReadingTest
 import org.jetbrains.kotlin.checkers.AbstractForeignJava8AnnotationsTest
 import org.jetbrains.kotlin.checkers.javac.AbstractJavacForeignJava8AnnotationsTest
-import org.jetbrains.kotlin.generators.tests.generator.testGroup
+import org.jetbrains.kotlin.generators.tests.generator.testGroupSuite
 import org.jetbrains.kotlin.jvm.compiler.AbstractLoadJava8Test
 import org.jetbrains.kotlin.jvm.compiler.AbstractLoadJava8WithPsiClassReadingTest
 import org.jetbrains.kotlin.jvm.compiler.javac.AbstractLoadJava8UsingJavacTest
 import org.jetbrains.kotlin.resolve.calls.AbstractEnhancedSignaturesResolvedCallsTest
 
-fun main() {
+fun main(args: Array<String>) {
     System.setProperty("java.awt.headless", "true")
 
-    testGroup("compiler/tests-java8/tests", "compiler/testData") {
+    testGroupSuite(args) {
-        testClass<AbstractForeignJava8AnnotationsTest> {
+        testGroup("compiler/tests-java8/tests", "compiler/testData") {
-            model("foreignAnnotationsJava8/tests")
+            testClass<AbstractForeignJava8AnnotationsTest> {
-        }
+                model("foreignAnnotationsJava8/tests")
+            }
 
-        testClass<AbstractJavacForeignJava8AnnotationsTest> {
+            testClass<AbstractJavacForeignJava8AnnotationsTest> {
-            model("foreignAnnotationsJava8/tests")
+                model("foreignAnnotationsJava8/tests")
-        }
+            }
 
-        testClass<AbstractForeignJava8AnnotationsNoAnnotationInClasspathTest> {
+            testClass<AbstractForeignJava8AnnotationsNoAnnotationInClasspathTest> {
-            model("foreignAnnotationsJava8/tests")
+                model("foreignAnnotationsJava8/tests")
-        }
+            }
 
-        testClass<AbstractForeignJava8AnnotationsNoAnnotationInClasspathWithPsiClassReadingTest> {
+            testClass<AbstractForeignJava8AnnotationsNoAnnotationInClasspathWithPsiClassReadingTest> {
-            model("foreignAnnotationsJava8/tests")
+                model("foreignAnnotationsJava8/tests")
-        }
+            }
 
-        testClass<AbstractLoadJava8Test> {
+            testClass<AbstractLoadJava8Test> {
-            model("loadJava8/compiledJava", extension = "java", testMethod = "doTestCompiledJava")
+                model("loadJava8/compiledJava", extension = "java", testMethod = "doTestCompiledJava")
-            model("loadJava8/sourceJava", extension = "java", testMethod = "doTestSourceJava")
+                model("loadJava8/sourceJava", extension = "java", testMethod = "doTestSourceJava")
-        }
+            }
 
-        testClass<AbstractLoadJava8UsingJavacTest> {
+            testClass<AbstractLoadJava8UsingJavacTest> {
-            model("loadJava8/compiledJava", extension = "java", testMethod = "doTestCompiledJava")
+                model("loadJava8/compiledJava", extension = "java", testMethod = "doTestCompiledJava")
-            model("loadJava8/sourceJava", extension = "java", testMethod = "doTestSourceJava")
+                model("loadJava8/sourceJava", extension = "java", testMethod = "doTestSourceJava")
-        }
+            }
 
-        testClass<AbstractLoadJava8WithPsiClassReadingTest> {
+            testClass<AbstractLoadJava8WithPsiClassReadingTest> {
-            model("loadJava8/compiledJava", extension = "java", testMethod = "doTestCompiledJava")
+                model("loadJava8/compiledJava", extension = "java", testMethod = "doTestCompiledJava")
-        }
+            }
 
-        testClass<AbstractEnhancedSignaturesResolvedCallsTest> {
+            testClass<AbstractEnhancedSignaturesResolvedCallsTest> {
-            model("resolvedCalls/enhancedSignatures")
+                model("resolvedCalls/enhancedSignatures")
+            }
         }
     }
 }

compiler/tests-spec/tests/org/jetbrains/kotlin/spec/utils/tasks/GenerateSpecTests.kt

@@ -5,7 +5,7 @@
 
 package org.jetbrains.kotlin.spec.utils.tasks
 
-import org.jetbrains.kotlin.generators.tests.generator.testGroup
+import org.jetbrains.kotlin.generators.tests.generator.testGroupSuite
 import org.jetbrains.kotlin.spec.checkers.AbstractDiagnosticsTestSpec
 import org.jetbrains.kotlin.spec.checkers.AbstractFirDiagnosticsTestSpec
 import org.jetbrains.kotlin.spec.codegen.AbstractBlackBoxCodegenTestSpec
@@ -38,32 +38,34 @@ fun detectDirsWithTestsMapFileOnly(dirName: String): List<String> {
 fun generateTests() {
     val excludedFirTestdataPattern = "^(.+)\\.fir\\.kts?\$"
 
-    testGroup(SPEC_TEST_PATH, SPEC_TESTDATA_PATH) {
+    testGroupSuite {
-        testClass<AbstractDiagnosticsTestSpec> {
+        testGroup(SPEC_TEST_PATH, SPEC_TESTDATA_PATH) {
-            model(
+            testClass<AbstractDiagnosticsTestSpec> {
-                "diagnostics",
+                model(
-                excludeDirs = listOf("helpers") + detectDirsWithTestsMapFileOnly("diagnostics"),
+                    "diagnostics",
-                excludedPattern = excludedFirTestdataPattern
+                    excludeDirs = listOf("helpers") + detectDirsWithTestsMapFileOnly("diagnostics"),
-            )
+                    excludedPattern = excludedFirTestdataPattern
-        }
+                )
+            }
 
-        testClass<AbstractFirDiagnosticsTestSpec> {
+            testClass<AbstractFirDiagnosticsTestSpec> {
-            model(
+                model(
-                "diagnostics",
+                    "diagnostics",
-                excludeDirs = listOf("helpers") + detectDirsWithTestsMapFileOnly("diagnostics"),
+                    excludeDirs = listOf("helpers") + detectDirsWithTestsMapFileOnly("diagnostics"),
-                excludedPattern = excludedFirTestdataPattern
+                    excludedPattern = excludedFirTestdataPattern
-            )
+                )
-        }
+            }
 
-        testClass<AbstractParsingTestSpec> {
+            testClass<AbstractParsingTestSpec> {
-            model(
+                model(
-                relativeRootPath = "psi",
+                    relativeRootPath = "psi",
-                testMethod = "doParsingTest",
+                    testMethod = "doParsingTest",
-                excludeDirs = listOf("helpers", "templates") + detectDirsWithTestsMapFileOnly("psi")
+                    excludeDirs = listOf("helpers", "templates") + detectDirsWithTestsMapFileOnly("psi")
-            )
+                )
-        }
+            }
-        testClass<AbstractBlackBoxCodegenTestSpec> {
+            testClass<AbstractBlackBoxCodegenTestSpec> {
-            model("codegen/box", excludeDirs = listOf("helpers", "templates") + detectDirsWithTestsMapFileOnly("codegen/box"))
+                model("codegen/box", excludeDirs = listOf("helpers", "templates") + detectDirsWithTestsMapFileOnly("codegen/box"))
+            }
         }
     }
 }

core/descriptors.runtime/tests/org/jetbrains/kotlin/generators/tests/GenerateRuntimeDescriptorTests.kt

@@ -16,21 +16,23 @@
 
 package org.jetbrains.kotlin.generators.tests
 
-import org.jetbrains.kotlin.generators.tests.generator.testGroup
+import org.jetbrains.kotlin.generators.tests.generator.testGroupSuite
 import org.jetbrains.kotlin.jvm.runtime.AbstractJvm8RuntimeDescriptorLoaderTest
 import org.jetbrains.kotlin.jvm.runtime.AbstractJvmRuntimeDescriptorLoaderTest
 
-fun main() {
+fun main(args: Array<String>) {
     System.setProperty("java.awt.headless", "true")
 
-    testGroup("core/descriptors.runtime/tests", "compiler/testData") {
+    testGroupSuite(args) {
-        testClass<AbstractJvmRuntimeDescriptorLoaderTest> {
+        testGroup("core/descriptors.runtime/tests", "compiler/testData") {
-            model("loadJava/compiledKotlin")
+            testClass<AbstractJvmRuntimeDescriptorLoaderTest> {
-            model("loadJava/compiledJava", extension = "java", excludeDirs = listOf("sam", "kotlinSignature/propagation"))
+                model("loadJava/compiledKotlin")
-        }
+                model("loadJava/compiledJava", extension = "java", excludeDirs = listOf("sam", "kotlinSignature/propagation"))
+            }
 
-        testClass<AbstractJvm8RuntimeDescriptorLoaderTest> {
+            testClass<AbstractJvm8RuntimeDescriptorLoaderTest> {
-            model("loadJava8/compiledJava", extension = "java")
+                model("loadJava8/compiledJava", extension = "java")
+            }
         }
     }
 }

generators/main/GeneratorsFileUtil.kt

@@ -24,7 +24,7 @@ object GeneratorsFileUtil {
                 throw IllegalStateException("Cannot create directory: $parentFile")
             }
         }
-        if (checkFileIgnoringLineSeparators(file, newText)) {
+        if (!isFileContentChangedIgnoringLineSeparators(file, newText)) {
             if (logNotChanged) {
                 println("Not changed: " + file.absolutePath)
             }
@@ -42,12 +42,12 @@ object GeneratorsFileUtil {
         println()
     }
 
-    private fun checkFileIgnoringLineSeparators(file: File, content: String): Boolean {
+    fun isFileContentChangedIgnoringLineSeparators(file: File, content: String): Boolean {
         val currentContent: String = try {
             StringUtil.convertLineSeparators(file.readText(Charsets.UTF_8))
         } catch (ignored: Throwable) {
-            return false
+            return true
         }
-        return StringUtil.convertLineSeparators(content) == currentContent
+        return StringUtil.convertLineSeparators(content) != currentContent
     }
 }

generators/test-generator/tests/org/jetbrains/kotlin/generators/tests/generator/TestGenerationDSL.kt

@@ -6,6 +6,8 @@
 package org.jetbrains.kotlin.generators.tests.generator
 
 import junit.framework.TestCase
+import org.jetbrains.kotlin.generators.tests.generator.InconsistencyChecker.Companion.hasDryRunArg
+import org.jetbrains.kotlin.generators.tests.generator.InconsistencyChecker.Companion.inconsistencyChecker
 import org.jetbrains.kotlin.test.TargetBackend
 import java.io.File
 import java.util.*
@@ -16,7 +18,8 @@ class TestGroup(
     val testDataRoot: String,
     val testRunnerMethodName: String,
     val additionalRunnerArguments: List<String> = emptyList(),
-    val annotations: List<AnnotationModel> = emptyList()
+    val annotations: List<AnnotationModel> = emptyList(),
+    private val dryRun: Boolean = false
 ) {
     inline fun <reified T : TestCase> testClass(
         suiteTestClassName: String = getDefaultSuiteTestClassName(T::class.java.simpleName),
@@ -34,13 +37,16 @@ class TestGroup(
         annotations: List<AnnotationModel> = emptyList(),
         init: TestClass.() -> Unit
     ) {
-        TestGenerator(
+        val testGenerator = TestGenerator(
             testsRoot,
             suiteTestClassName,
             baseTestClassName,
             TestClass(annotations).apply(init).testModels,
             useJunit4
-        ).generateAndSave()
+        )
+        if (testGenerator.generateAndSave(dryRun)) {
+            inconsistencyChecker(dryRun).add(testGenerator.testSourceFilePath)
+        }
     }
 
     inner class TestClass(val annotations: List<AnnotationModel>) {
@@ -85,14 +91,67 @@ class TestGroup(
     }
 }
 
-fun testGroup(
+fun testGroupSuite(
-    testsRoot: String,
+    args: Array<String>,
-    testDataRoot: String,
+    init: TestGroupSuite.() -> Unit
-    testRunnerMethodName: String = RunTestMethodModel.METHOD_NAME,
-    additionalRunnerArguments: List<String> = emptyList(),
-    init: TestGroup.() -> Unit
 ) {
-    TestGroup(testsRoot, testDataRoot, testRunnerMethodName, additionalRunnerArguments).init()
+    testGroupSuite(hasDryRunArg(args), init)
+}
+
+fun testGroupSuite(
+    dryRun: Boolean = false,
+    init: TestGroupSuite.() -> Unit
+) {
+    TestGroupSuite(dryRun).init()
+}
+
+class TestGroupSuite(private val dryRun: Boolean) {
+    fun testGroup(
+        testsRoot: String,
+        testDataRoot: String,
+        testRunnerMethodName: String = RunTestMethodModel.METHOD_NAME,
+        additionalRunnerArguments: List<String> = emptyList(),
+        init: TestGroup.() -> Unit
+    ) {
+        TestGroup(
+            testsRoot,
+            testDataRoot,
+            testRunnerMethodName,
+            additionalRunnerArguments,
+            dryRun = dryRun
+        ).init()
+    }
+}
+
+interface InconsistencyChecker {
+    fun add(affectedFile: String)
+
+    val affectedFiles: List<String>
+
+    companion object {
+        fun hasDryRunArg(args: Array<String>) = args.any { it == "dryRun" }
+
+        fun inconsistencyChecker(dryRun: Boolean) = if (dryRun) DefaultInconsistencyChecker else EmptyInconsistencyChecker
+    }
+}
+
+object DefaultInconsistencyChecker : InconsistencyChecker {
+    private val files = mutableListOf<String>()
+
+    override fun add(affectedFile: String) {
+        files.add(affectedFile)
+    }
+
+    override val affectedFiles: List<String>
+        get() = files
+}
+
+object EmptyInconsistencyChecker : InconsistencyChecker {
+    override fun add(affectedFile: String) {
+    }
+
+    override val affectedFiles: List<String>
+        get() = emptyList()
 }
 
 fun getDefaultSuiteTestClassName(baseTestClassName: String): String {

generators/test-generator/tests/org/jetbrains/kotlin/generators/tests/generator/TestGenerator.kt

@@ -8,7 +8,6 @@ package org.jetbrains.kotlin.generators.tests.generator
 import org.jetbrains.kotlin.generators.util.GeneratorsFileUtil
 import org.jetbrains.kotlin.test.JUnit3RunnerWithInners
 import org.jetbrains.kotlin.test.KotlinTestUtils
-import org.jetbrains.kotlin.test.TargetBackend
 import org.jetbrains.kotlin.test.TestMetadata
 import org.jetbrains.kotlin.utils.Printer
 import org.junit.Test
@@ -31,7 +30,7 @@ class TestGenerator(
     private val baseTestClassName: String
     private val testClassModels: Collection<TestClassModel>
     private val useJunit4: Boolean
-    private val testSourceFilePath: String
+    internal val testSourceFilePath: String
 
     init {
         this.baseTestClassPackage = baseTestClassFqName.substringBeforeLast('.', "")
@@ -48,18 +47,33 @@ class TestGenerator(
         }
     }
 
+    /**
+     * @return true if a new file is generated
+     */
     @Throws(IOException::class)
-    fun generateAndSave() {
+    fun generateAndSave(dryRun: Boolean): Boolean {
+        val generatedCode = generate()
+
+        val testSourceFile = File(testSourceFilePath)
+        val changed =
+            GeneratorsFileUtil.isFileContentChangedIgnoringLineSeparators(testSourceFile, generatedCode)
+        if (!dryRun) {
+            GeneratorsFileUtil.writeFileIfContentChanged(testSourceFile, generatedCode, false)
+        }
+        return changed
+    }
+
+    private fun generate(): String {
         val out = StringBuilder()
         val p = Printer(out)
 
         val year = GregorianCalendar()[Calendar.YEAR]
         p.println(
             """/*
-            | * Copyright 2010-$year JetBrains s.r.o. and Kotlin Programming Language contributors.
+                | * Copyright 2010-$year JetBrains s.r.o. and Kotlin Programming Language contributors.
-            | * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+                | * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
-            | */
+                | */
-            |""".trimMargin()
+                |""".trimMargin()
         )
         p.println("package ", suiteClassPackage, ";")
         p.println()
@@ -126,9 +140,7 @@ class TestGenerator(
         }
 
         generateTestClass(p, model, false)
-
+        return out.toString()
-        val testSourceFile = File(testSourceFilePath)
-        GeneratorsFileUtil.writeFileIfContentChanged(testSourceFile, out.toString(), false)
     }
 
     private fun generateTestClass(p: Printer, testClassModel: TestClassModel, isStatic: Boolean) {

js/js.tests/test/org/jetbrains/kotlin/generators/tests/GenerateJsTests.kt

@@ -5,7 +5,7 @@
 
 package org.jetbrains.kotlin.generators.tests
 
-import org.jetbrains.kotlin.generators.tests.generator.testGroup
+import org.jetbrains.kotlin.generators.tests.generator.testGroupSuite
 import org.jetbrains.kotlin.js.test.AbstractDceTest
 import org.jetbrains.kotlin.js.test.AbstractJsLineNumberTest
 import org.jetbrains.kotlin.js.test.es6.semantics.*
@@ -14,90 +14,92 @@ import org.jetbrains.kotlin.js.test.semantics.*
 import org.jetbrains.kotlin.js.test.wasm.semantics.AbstractIrWasmBoxWasmTest
 import org.jetbrains.kotlin.test.TargetBackend
 
-fun main() {
+fun main(args: Array<String>) {
     System.setProperty("java.awt.headless", "true")
 
     // TODO: repair these tests
     //generateTestDataForReservedWords()
 
-    testGroup("js/js.tests/test", "js/js.translator/testData", testRunnerMethodName = "runTest0") {
+    testGroupSuite(args) {
-        testClass<AbstractBoxJsTest> {
+        testGroup("js/js.tests/test", "js/js.translator/testData", testRunnerMethodName = "runTest0") {
-            model("box/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS)
+            testClass<AbstractBoxJsTest> {
+                model("box/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS)
+            }
+
+            testClass<AbstractIrBoxJsTest> {
+                model("box/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS_IR)
+            }
+
+            testClass<AbstractIrBoxJsES6Test> {
+                model("box/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS_IR_ES6)
+            }
+
+            testClass<AbstractIrJsTypeScriptExportTest> {
+                model("typescript-export/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS_IR)
+            }
+
+            testClass<AbstractIrJsTypeScriptExportES6Test> {
+                model("typescript-export/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS_IR_ES6)
+            }
+
+            testClass<AbstractLegacyJsTypeScriptExportTest> {
+                model("typescript-export/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS)
+            }
+
+
+            testClass<AbstractSourceMapGenerationSmokeTest> {
+                model("sourcemap/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS)
+            }
+
+            testClass<AbstractOutputPrefixPostfixTest> {
+                model("outputPrefixPostfix/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS)
+            }
+
+            testClass<AbstractDceTest> {
+                model("dce/", pattern = "(.+)\\.js", targetBackend = TargetBackend.JS)
+            }
+
+            testClass<AbstractJsLineNumberTest> {
+                model("lineNumbers/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS)
+            }
+
+            testClass<AbstractIrWasmBoxWasmTest> {
+                model("wasmBox", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.WASM)
+            }
+
+            testClass<AbstractIrWasmBoxJsTest> {
+                model("wasmBox", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS_IR)
+            }
         }
 
-        testClass<AbstractIrBoxJsTest> {
+        testGroup("js/js.tests/test", "compiler/testData", testRunnerMethodName = "runTest0") {
-            model("box/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS_IR)
+            testClass<AbstractJsCodegenBoxTest> {
-        }
+                model("codegen/box", targetBackend = TargetBackend.JS)
+            }
 
-        testClass<AbstractIrBoxJsES6Test> {
+            testClass<AbstractIrJsCodegenBoxTest> {
-            model("box/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS_IR_ES6)
+                model("codegen/box", targetBackend = TargetBackend.JS_IR)
-        }
+            }
 
-        testClass<AbstractIrJsTypeScriptExportTest> {
+            testClass<AbstractIrJsCodegenBoxES6Test> {
-            model("typescript-export/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS_IR)
+                model("codegen/box", targetBackend = TargetBackend.JS_IR_ES6)
-        }
+            }
 
-        testClass<AbstractIrJsTypeScriptExportES6Test> {
+            testClass<AbstractJsCodegenInlineTest> {
-            model("typescript-export/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS_IR_ES6)
+                model("codegen/boxInline/", targetBackend = TargetBackend.JS)
-        }
+            }
 
-        testClass<AbstractLegacyJsTypeScriptExportTest> {
+            testClass<AbstractIrJsCodegenInlineTest> {
-            model("typescript-export/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS)
+                model("codegen/boxInline/", targetBackend = TargetBackend.JS_IR)
-        }
+            }
 
+            testClass<AbstractIrJsCodegenInlineES6Test> {
+                model("codegen/boxInline/", targetBackend = TargetBackend.JS_IR_ES6)
+            }
 
-        testClass<AbstractSourceMapGenerationSmokeTest> {
+            testClass<AbstractJsLegacyPrimitiveArraysBoxTest> {
-            model("sourcemap/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS)
+                model("codegen/box/arrays", targetBackend = TargetBackend.JS)
-        }
+            }
-
-        testClass<AbstractOutputPrefixPostfixTest> {
-            model("outputPrefixPostfix/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS)
-        }
-
-        testClass<AbstractDceTest> {
-            model("dce/", pattern = "(.+)\\.js", targetBackend = TargetBackend.JS)
-        }
-
-        testClass<AbstractJsLineNumberTest> {
-            model("lineNumbers/", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS)
-        }
-
-        testClass<AbstractIrWasmBoxWasmTest> {
-            model("wasmBox", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.WASM)
-        }
-
-        testClass<AbstractIrWasmBoxJsTest> {
-            model("wasmBox", pattern = "^([^_](.+))\\.kt$", targetBackend = TargetBackend.JS_IR)
-        }
-    }
-
-    testGroup("js/js.tests/test", "compiler/testData", testRunnerMethodName = "runTest0") {
-        testClass<AbstractJsCodegenBoxTest> {
-            model("codegen/box", targetBackend = TargetBackend.JS)
-        }
-
-        testClass<AbstractIrJsCodegenBoxTest> {
-            model("codegen/box", targetBackend = TargetBackend.JS_IR)
-        }
-
-        testClass<AbstractIrJsCodegenBoxES6Test> {
-            model("codegen/box", targetBackend = TargetBackend.JS_IR_ES6)
-        }
-
-        testClass<AbstractJsCodegenInlineTest> {
-            model("codegen/boxInline/", targetBackend = TargetBackend.JS)
-        }
-
-        testClass<AbstractIrJsCodegenInlineTest> {
-            model("codegen/boxInline/", targetBackend = TargetBackend.JS_IR)
-        }
-
-        testClass<AbstractIrJsCodegenInlineES6Test> {
-            model("codegen/boxInline/", targetBackend = TargetBackend.JS_IR_ES6)
-        }
-
-        testClass<AbstractJsLegacyPrimitiveArraysBoxTest> {
-            model("codegen/box/arrays", targetBackend = TargetBackend.JS)
         }
     }
 }

plugins/pill/generate-all-tests/test/org/jetbrains/kotlin/pill/generateAllTests/Main.java

@@ -6,13 +6,24 @@
 package org.jetbrains.kotlin.pill.generateAllTests;
 
 import org.jetbrains.kotlin.generators.tests.*;
+import org.jetbrains.kotlin.generators.tests.generator.InconsistencyChecker;
+
+import java.util.List;
 
 public class Main {
     public static void main(String[] args) {
-        GenerateCompilerTestsKt.main();
+        GenerateCompilerTestsKt.main(args);
-        GenerateTestsKt.main();
+        GenerateTestsKt.main(args);
-        GenerateJsTestsKt.main();
+        GenerateJsTestsKt.main(args);
-        GenerateJava8TestsKt.main();
+        GenerateJava8TestsKt.main(args);
-        GenerateRuntimeDescriptorTestsKt.main();
+        GenerateRuntimeDescriptorTestsKt.main(args);
+
+        boolean dryRun = InconsistencyChecker.Companion.hasDryRunArg(args);
+        List<String> affectedFiles = InconsistencyChecker.Companion.inconsistencyChecker(dryRun).getAffectedFiles();
+        int size = affectedFiles.size();
+        if (size > 0) {
+            throw new IllegalStateException("There " + (size == 1 ? "is a test" : "are " + size + " tests") + " to be regenerated:\n"
+                                            + String.join("\n", affectedFiles));
+        }
     }
 }