diff --git a/compiler/fir/resolve/tests/org/jetbrains/kotlin/fir/AbstractFirResolveTestCaseWithStdlib.kt b/compiler/fir/resolve/tests/org/jetbrains/kotlin/fir/AbstractFirResolveTestCaseWithStdlib.kt
index 3b821d9acfe..4e64e4b2d27 100644
--- a/compiler/fir/resolve/tests/org/jetbrains/kotlin/fir/AbstractFirResolveTestCaseWithStdlib.kt
+++ b/compiler/fir/resolve/tests/org/jetbrains/kotlin/fir/AbstractFirResolveTestCaseWithStdlib.kt
@@ -9,7 +9,8 @@ import org.jetbrains.kotlin.test.ConfigurationKind
 import org.jetbrains.kotlin.test.TestJdkKind
 
 abstract class AbstractFirDiagnosticsWithStdlibTest : AbstractFirDiagnosticsTest() {
-    override fun getConfigurationKind(): ConfigurationKind {
+
+    override fun extractConfigurationKind(files: List<TestFile>): ConfigurationKind {
         return ConfigurationKind.ALL
     }
 }
\ No newline at end of file
diff --git a/compiler/testData/diagnostics/testWithModifiedMockJdk/newStringMethods.kt b/compiler/testData/diagnostics/testWithModifiedMockJdk/newStringMethods.kt
index 53a19e64041..c6a87538327 100644
--- a/compiler/testData/diagnostics/testWithModifiedMockJdk/newStringMethods.kt
+++ b/compiler/testData/diagnostics/testWithModifiedMockJdk/newStringMethods.kt
@@ -1,5 +1,6 @@
 // !CHECK_TYPE
 // SKIP_TXT
+// WITH_RUNTIME
 
 fun foo(s: String) {
     s.isBlank()
diff --git a/compiler/testData/diagnostics/tests/when/withSubjectVariable/unsupportedVariableDeclarationsInWhenSubject.fir.kt b/compiler/testData/diagnostics/tests/when/withSubjectVariable/unsupportedVariableDeclarationsInWhenSubject.fir.kt
index 6268016b679..bdfbe8f64f0 100644
--- a/compiler/testData/diagnostics/tests/when/withSubjectVariable/unsupportedVariableDeclarationsInWhenSubject.fir.kt
+++ b/compiler/testData/diagnostics/tests/when/withSubjectVariable/unsupportedVariableDeclarationsInWhenSubject.fir.kt
@@ -1,6 +1,5 @@
 // !LANGUAGE: +VariableDeclarationInWhenSubject
 // !DIAGNOSTICS: -UNUSED_VARIABLE -UNUSED_PARAMETER
-// WITH_RUNTIME
 
 fun foo(): Any = 42
 fun String.bar(): Any = 42
diff --git a/compiler/testData/diagnostics/tests/when/withSubjectVariable/unsupportedVariableDeclarationsInWhenSubject.kt b/compiler/testData/diagnostics/tests/when/withSubjectVariable/unsupportedVariableDeclarationsInWhenSubject.kt
index 0fc506ca558..118a402169c 100644
--- a/compiler/testData/diagnostics/tests/when/withSubjectVariable/unsupportedVariableDeclarationsInWhenSubject.kt
+++ b/compiler/testData/diagnostics/tests/when/withSubjectVariable/unsupportedVariableDeclarationsInWhenSubject.kt
@@ -1,6 +1,5 @@
 // !LANGUAGE: +VariableDeclarationInWhenSubject
 // !DIAGNOSTICS: -UNUSED_VARIABLE -UNUSED_PARAMETER
-// WITH_RUNTIME
 
 fun foo(): Any = 42
 fun String.bar(): Any = 42
diff --git a/compiler/testData/diagnostics/testsWithJava9/kt11167.kt b/compiler/testData/diagnostics/testsWithJava9/kt11167.kt
index 18b016c0f84..56696a9babc 100644
--- a/compiler/testData/diagnostics/testsWithJava9/kt11167.kt
+++ b/compiler/testData/diagnostics/testsWithJava9/kt11167.kt
@@ -1,3 +1,4 @@
+//WITH_RUNTIME
 import java.util.stream.IntStream
 
 fun foo(s: IntStream) {
diff --git a/compiler/testData/foreignAnnotationsJava8/tests/jsr305/typeArguments.kt b/compiler/testData/foreignAnnotationsJava8/tests/jsr305/typeArguments.kt
index 563e5ed7c72..ee376f2237e 100644
--- a/compiler/testData/foreignAnnotationsJava8/tests/jsr305/typeArguments.kt
+++ b/compiler/testData/foreignAnnotationsJava8/tests/jsr305/typeArguments.kt
@@ -2,7 +2,7 @@
 // !WITH_NEW_INFERENCE
 // RENDER_PACKAGE: test
 // SKIP_COMPILED_JAVA
-
+// WITH_RUNTIME
 // FILE: spr/Nullable.java
 
 package spr;
diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/cfg/AbstractDiagnosticsWithModifiedMockJdkTest.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/cfg/AbstractDiagnosticsWithModifiedMockJdkTest.kt
index 5191cb9132e..c4d63444e7a 100644
--- a/compiler/tests-common/tests/org/jetbrains/kotlin/cfg/AbstractDiagnosticsWithModifiedMockJdkTest.kt
+++ b/compiler/tests-common/tests/org/jetbrains/kotlin/cfg/AbstractDiagnosticsWithModifiedMockJdkTest.kt
@@ -16,14 +16,9 @@
 package org.jetbrains.kotlin.cfg
 
 import org.jetbrains.kotlin.checkers.AbstractDiagnosticsTest
-import org.jetbrains.kotlin.test.ConfigurationKind
 import org.jetbrains.kotlin.test.TestJdkKind
 
 abstract class AbstractDiagnosticsWithModifiedMockJdkTest : AbstractDiagnosticsTest() {
-    override fun getConfigurationKind(): ConfigurationKind {
-        return ConfigurationKind.ALL
-    }
-
     override fun getTestJdkKind(files: List<TestFile>): TestJdkKind {
         return TestJdkKind.MODIFIED_MOCK_JDK
     }
diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsTestWithStdLib.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsTestWithStdLib.kt
index 38eb1237105..e6614db84c8 100644
--- a/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsTestWithStdLib.kt
+++ b/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsTestWithStdLib.kt
@@ -8,7 +8,8 @@ import org.jetbrains.kotlin.test.ConfigurationKind
 import java.io.File
 
 abstract class AbstractDiagnosticsTestWithStdLib : AbstractDiagnosticsTest() {
-    override fun getConfigurationKind(): ConfigurationKind {
+
+    override fun extractConfigurationKind(files: List<TestFile>): ConfigurationKind {
         return ConfigurationKind.NO_KOTLIN_REFLECT
     }
 
diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsWithExplicitApi.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsWithExplicitApi.kt
index 99e9b14a208..a5fb332ffdd 100644
--- a/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsWithExplicitApi.kt
+++ b/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsWithExplicitApi.kt
@@ -9,10 +9,6 @@ import org.jetbrains.kotlin.config.*
 import org.jetbrains.kotlin.test.ConfigurationKind
 
 abstract class AbstractDiagnosticsWithExplicitApi : AbstractDiagnosticsTest() {
-    override fun getConfigurationKind(): ConfigurationKind {
-        return ConfigurationKind.NO_KOTLIN_REFLECT
-    }
-
     override fun defaultLanguageVersionSettings(): LanguageVersionSettings =
         CompilerTestLanguageVersionSettings(
             DEFAULT_DIAGNOSTIC_TESTS_FEATURES,
diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsWithJdk9Test.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsWithJdk9Test.kt
index b8e37c15281..f955621639d 100644
--- a/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsWithJdk9Test.kt
+++ b/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsWithJdk9Test.kt
@@ -15,11 +15,9 @@
  */
 package org.jetbrains.kotlin.checkers
 
-import org.jetbrains.kotlin.test.ConfigurationKind
 import org.jetbrains.kotlin.test.TestJdkKind
 
 abstract class AbstractDiagnosticsWithJdk9Test : AbstractDiagnosticsTest() {
-    override fun getConfigurationKind(): ConfigurationKind = ConfigurationKind.ALL
 
     override fun getTestJdkKind(files: List<TestFile>): TestJdkKind {
         return TestJdkKind.FULL_JDK_9
diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsWithUnsignedTypes.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsWithUnsignedTypes.kt
index a43dd3f7254..99f536df4ac 100644
--- a/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsWithUnsignedTypes.kt
+++ b/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractDiagnosticsWithUnsignedTypes.kt
@@ -12,7 +12,8 @@ import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.test.ConfigurationKind
 
 abstract class AbstractDiagnosticsWithUnsignedTypes : AbstractDiagnosticsTest() {
-    override fun getConfigurationKind(): ConfigurationKind {
+
+    override fun extractConfigurationKind(files: List<TestFile>): ConfigurationKind {
         return ConfigurationKind.NO_KOTLIN_REFLECT
     }
 
diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractForeignAnnotationsTest.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractForeignAnnotationsTest.kt
index 73098abd814..ab334abd540 100644
--- a/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractForeignAnnotationsTest.kt
+++ b/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/AbstractForeignAnnotationsTest.kt
@@ -52,8 +52,6 @@ abstract class AbstractForeignAnnotationsTest : AbstractDiagnosticsTest() {
             ForTestCompileRuntime.jvmAnnotationsForTests()
     )
 
-    override fun getConfigurationKind(): ConfigurationKind = ConfigurationKind.ALL
-
     open protected val annotationsPath: String
         get() = FOREIGN_ANNOTATIONS_SOURCES_PATH
 
diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/KotlinMultiFileTestWithJava.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/KotlinMultiFileTestWithJava.kt
index 65edf722fb7..2673f0c2d19 100644
--- a/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/KotlinMultiFileTestWithJava.kt
+++ b/compiler/tests-common/tests/org/jetbrains/kotlin/checkers/KotlinMultiFileTestWithJava.kt
@@ -79,8 +79,6 @@ abstract class KotlinMultiFileTestWithJava<M : KotlinBaseTest.TestModule, F : Ko
         setupEnvironment(environment)
     }
 
-    protected open fun getConfigurationKind() = ConfigurationKind.JDK_ONLY
-
     private fun getClasspath(file: File): List<File> {
         val result: MutableList<File> = ArrayList()
         result.add(KotlinTestUtils.getAnnotationsJar())
diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/fir/AbstractFirOldFrontendDiagnosticsTestWithStdlib.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/fir/AbstractFirOldFrontendDiagnosticsTestWithStdlib.kt
index aadf73a1738..ad7f2d8e4d8 100644
--- a/compiler/tests-common/tests/org/jetbrains/kotlin/fir/AbstractFirOldFrontendDiagnosticsTestWithStdlib.kt
+++ b/compiler/tests-common/tests/org/jetbrains/kotlin/fir/AbstractFirOldFrontendDiagnosticsTestWithStdlib.kt
@@ -8,7 +8,7 @@ package org.jetbrains.kotlin.fir
 import org.jetbrains.kotlin.test.ConfigurationKind
 
 abstract class AbstractFirOldFrontendDiagnosticsTestWithStdlib : AbstractFirOldFrontendDiagnosticsTest() {
-    override fun getConfigurationKind(): ConfigurationKind {
+    override fun extractConfigurationKind(files: List<TestFile>): ConfigurationKind {
         return ConfigurationKind.NO_KOTLIN_REFLECT
     }
 }
\ No newline at end of file
diff --git a/compiler/tests-spec/tests/org/jetbrains/kotlin/spec/checkers/AbstractDiagnosticsTestSpec.kt b/compiler/tests-spec/tests/org/jetbrains/kotlin/spec/checkers/AbstractDiagnosticsTestSpec.kt
index 91e257ce7ec..50b645f57ab 100644
--- a/compiler/tests-spec/tests/org/jetbrains/kotlin/spec/checkers/AbstractDiagnosticsTestSpec.kt
+++ b/compiler/tests-spec/tests/org/jetbrains/kotlin/spec/checkers/AbstractDiagnosticsTestSpec.kt
@@ -47,7 +47,9 @@ abstract class AbstractDiagnosticsTestSpec : org.jetbrains.kotlin.checkers.Abstr
         }
     }
 
-    override fun getConfigurationKind() = ConfigurationKind.ALL
+    override fun extractConfigurationKind(files: List<TestFile>): ConfigurationKind {
+        return ConfigurationKind.ALL
+    }
 
     override fun skipDescriptorsValidation() = skipDescriptors