[Gradle] Add integration tests on new diagnostics infrastructure

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BaseGradleIT.kt

@@ -283,6 +283,7 @@ abstract class BaseGradleIT {
         val withReports: List<BuildReportType> = emptyList(),
         val enableKpmModelMapping: Boolean? = null,
         val useDaemonFallbackStrategy: Boolean = false,
+        val useVerboseDiagnosticsReporting: Boolean = true,
     ) {
         val safeAndroidGradlePluginVersion: AGPVersion
             get() = androidGradlePluginVersion ?: error("AGP version is expected to be set")
@@ -938,6 +939,10 @@ abstract class BaseGradleIT {
             add("-Dorg.gradle.unsafe.configuration-cache=${options.configurationCache}")
             add("-Dorg.gradle.unsafe.configuration-cache-problems=${options.configurationCacheProblems.name.lowercase(Locale.getDefault())}")
 
+            if (options.useVerboseDiagnosticsReporting) {
+                add("-Pkotlin.internal.verboseDiagnostics=true")
+            }
+
             // Workaround: override a console type set in the user machine gradle.properties (since Gradle 4.3):
             add("--console=plain")
             //The feature of failing the build on deprecation warnings is introduced in gradle 5.6

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/mpp/MppDiagnosticsIt.kt

@@ -15,6 +15,16 @@ import kotlin.test.assertTrue
 
 @MppGradlePluginTests
 class MppDiagnosticsIt : KGPBaseTest() {
+    @GradleTest
+    fun testDiagnosticsRenderingSmoke(gradleVersion: GradleVersion) {
+        project("diagnosticsRenderingSmoke", gradleVersion) {
+            val expectedOutputFile = projectName.testProjectPath.resolve("expectedOutput.txt").toFile()
+            build {
+                assertEqualsToFile(expectedOutputFile, extractProjectsAndTheirVerboseDiagnostics())
+            }
+        }
+    }
+
     @GradleTest
     fun testDeprecatedProperties(gradleVersion: GradleVersion) {
         project("mppDeprecatedProperties", gradleVersion) {
@@ -38,19 +48,6 @@ class MppDiagnosticsIt : KGPBaseTest() {
         }
     }
 
-    @GradleTest
-    fun testCommonMainMustNotDependOnOtherSourceSets(gradleVersion: GradleVersion) {
-        project("commonMainDependsOnAnotherSourceSet", gradleVersion) {
-            build("tasks") {
-                assertOutputContains("w: 'commonMain' source set can't depend on other source sets.")
-            }
-
-            build("tasks", buildOptions = defaultBuildOptions.copy(freeArgs = listOf("-PcommonSourceSetDependsOnNothing"))) {
-                assertOutputDoesNotContain("w: 'commonMain' source set can't depend on other source sets.")
-            }
-        }
-    }
-
     @GradleTest
     fun testReportTargetsOfTheSamplePlatformAndWithTheSameAttributes(gradleVersion: GradleVersion) {
         project("new-mpp-lib-and-app/sample-lib-gradle-kotlin-dsl", gradleVersion) {

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/BuildOptions.kt

@@ -42,6 +42,7 @@ data class BuildOptions(
     val usePreciseOutputsBackup: Boolean? = null,
     val keepIncrementalCompilationCachesInMemory: Boolean? = null,
     val useDaemonFallbackStrategy: Boolean = false,
+    val verboseDiagnostics: Boolean = true,
 ) {
     val safeAndroidVersion: String
         get() = androidVersion ?: error("AGP version is expected to be set")
@@ -165,6 +166,10 @@ data class BuildOptions(
 
         arguments.add("-Pkotlin.daemon.useFallbackStrategy=$useDaemonFallbackStrategy")
 
+        if (verboseDiagnostics) {
+            arguments.add("-Pkotlin.internal.verboseDiagnostics=$verboseDiagnostics")
+        }
+
         arguments.addAll(freeArgs)
 
         return arguments.toList()
@@ -187,4 +192,3 @@ fun BuildOptions.suppressDeprecationWarningsSinceGradleVersion(
 ) = suppressDeprecationWarningsOn(reason) {
     currentGradleVersion >= GradleVersion.version(gradleVersion)
 }
-

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/assertionsUtils.kt

@@ -6,6 +6,12 @@
 package org.jetbrains.kotlin.gradle.testbase
 
 import org.gradle.testkit.runner.BuildResult
+import org.jetbrains.kotlin.test.testFramework.KtUsefulTestCase
+import org.jetbrains.kotlin.test.util.convertLineSeparators
+import org.jetbrains.kotlin.test.util.trimTrailingWhitespacesAndAddNewlineAtEOF
+import java.io.File
+import kotlin.test.assertEquals
+import kotlin.test.fail
 
 internal fun BuildResult.printBuildOutput() {
     println(
@@ -20,3 +26,26 @@ internal fun BuildResult.printBuildOutput() {
 }
 
 internal fun String.normalizeLineEndings(): String = replace("\n", System.lineSeparator())
+
+/*
+Ideally, this would've been just KotlinTestUtils.assertEqualsToFile, throwing FileComparisonException.
+Normally this is actually desired, because IDEA and TC provide special support for those exceptions with
+nice diff windows. However, due to shaded artifacts used in runtime of integration tests, actual thrown
+exception will have FQN like org.jetbrains.kotlin.com.intellij... . This not only breaks the tooling support,
+but also rendered diff will be trimmed, making working with such failures extremely inconvenient
+*/
+internal fun assertEqualsToFile(expectedFile: File, actualText: String) {
+    val textSanitized: String = actualText.trim().convertLineSeparators().trimTrailingWhitespacesAndAddNewlineAtEOF()
+
+    if (!expectedFile.exists()) {
+        if (KtUsefulTestCase.IS_UNDER_TEAMCITY) {
+            fail("Expected data file $expectedFile did not exist")
+        } else {
+            expectedFile.writeText(textSanitized)
+            fail("Expected data file did not exist. Generating: $expectedFile")
+        }
+    }
+    val expected: String = expectedFile.readText().convertLineSeparators().trimTrailingWhitespacesAndAddNewlineAtEOF()
+
+    assertEquals(expected, textSanitized, "Comparison failure")
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/diagnosticsAssertions.kt

@@ -0,0 +1,127 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.gradle.testbase
+
+import org.gradle.testkit.runner.BuildResult
+import org.jetbrains.kotlin.gradle.BaseGradleIT
+import org.jetbrains.kotlin.gradle.internals.VERBOSE_DIAGNOSTIC_SEPARATOR
+import org.jetbrains.kotlin.gradle.plugin.diagnostics.ToolingDiagnosticFactory
+import kotlin.test.assertNotNull
+import kotlin.test.assertNull
+import kotlin.test.assertTrue
+
+fun BaseGradleIT.CompiledProject.assertHasDiagnostic(diagnosticFactory: ToolingDiagnosticFactory, withSubstring: String? = null) {
+    output.assertHasDiagnostic(diagnosticFactory, withSubstring)
+}
+
+fun BaseGradleIT.CompiledProject.assertNoDiagnostic(diagnosticFactory: ToolingDiagnosticFactory) {
+    output.assertNoDiagnostic(diagnosticFactory)
+}
+
+fun String.assertHasDiagnostic(diagnosticFactory: ToolingDiagnosticFactory, withSubstring: String? = null) {
+    val diagnosticMessage = extractVerboselyRenderedDiagnostic(diagnosticFactory, this)
+    assertNotNull(diagnosticMessage) { "Diagnostic with id=${diagnosticFactory.id} not found" }
+    if (withSubstring != null) {
+        assertTrue(
+            withSubstring in diagnosticMessage,
+            "Diagnostic ${diagnosticFactory.id} doesn't have expected substring $withSubstring. " +
+                    "Actual diagnostic message:\n" +
+                    diagnosticMessage
+        )
+    }
+}
+
+fun String.assertNoDiagnostic(diagnosticFactory: ToolingDiagnosticFactory) {
+    val diagnosticMessage = extractVerboselyRenderedDiagnostic(diagnosticFactory, this)
+    assertNull(
+        diagnosticMessage,
+        "Diagnostic with id=${diagnosticFactory.id} was expected to be absent, but was reported. " +
+                "Actual diagnostic message: \n" +
+                diagnosticMessage
+    )
+}
+
+/**
+ * NB: Needs verbose mode of diagnostics, see [org.jetbrains.kotlin.gradle.plugin.PropertiesProvider.internalVerboseDiagnostics]
+ */
+fun BuildResult.extractProjectsAndTheirVerboseDiagnostics(): String = buildString {
+    var diagnosticStarted = false
+    for ((index, line) in output.lines().withIndex()) {
+        when {
+            line.trim() == VERBOSE_DIAGNOSTIC_SEPARATOR -> {
+                if (diagnosticStarted) {
+                    appendLine(line)
+                    appendLine()
+                    diagnosticStarted = false
+                } else {
+                    printBuildOutput()
+                    error("Unexpected end of diagnostic $line on line ${index + 1}")
+                }
+            }
+
+            DIAGNOSTIC_START_REGEX.matches(line) -> {
+                if (!diagnosticStarted) {
+                    appendLine(line)
+                    diagnosticStarted = true
+                } else {
+                    printBuildOutput()
+                    error(
+                        "Unexpected start of diagnostic $line on line ${index + 1}. The end of the previous diagnostic wasn't found yet"
+                    )
+                }
+            }
+
+            diagnosticStarted -> {
+                appendLine(line)
+            }
+
+            line.startsWith(CONFIGURE_PROJECT_PREFIX) -> {
+                appendLine() // additional empty line between projects
+                appendLine(line)
+            }
+        }
+    }
+}.trim()
+
+/*
+Expected format
+      w: [DIAGNOSTIC_ID | WARNING] first line of diagnostic's text
+ */
+private val DIAGNOSTIC_START_REGEX = """\s*[we]:\s*\[[^\[]*].*""".toRegex()
+
+
+/*
+ Expected format:
+
+       diagnosticStartIndex     diagnosticHeaderEndIndex
+        |                        |
+        v                        v
+     w: [MyDiagnosticId | WARNING] Some
+     Multiline
+         Text
+     #diagnostic-end
+     ^
+     |
+     diagnosticSeparatorStartIndex
+
+ Expected result for this case:
+     "Some
+     Multiline
+         Text"
+  */
+private fun extractVerboselyRenderedDiagnostic(diagnostic: ToolingDiagnosticFactory, fromText: String): String? {
+    val diagnosticStartIndex = fromText.indexOf("[${diagnostic.id}")
+    if (diagnosticStartIndex == -1) return null
+
+    val diagnosticHeaderEnd = fromText.indexOf("]", startIndex = diagnosticStartIndex)
+    val diagnosticMessageStart = diagnosticHeaderEnd + 1
+
+    val diagnosticSeparatorStartIndex = fromText.indexOf(VERBOSE_DIAGNOSTIC_SEPARATOR, startIndex = diagnosticStartIndex)
+    // NB: substring's endIndex is exclusive, which gives us exactly the message
+    return fromText.substring(diagnosticMessageStart, diagnosticSeparatorStartIndex).trim { it.isWhitespace() || it == '\n' }
+}
+
+private const val CONFIGURE_PROJECT_PREFIX = "> Configure project"

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/testbase/testDsl.kt

@@ -435,7 +435,7 @@ private fun setupProjectFromTestResources(
         }
 }
 
-private val String.testProjectPath: Path get() = Paths.get("src", "test", "resources", "testProject", this)
+internal val String.testProjectPath: Path get() = Paths.get("src", "test", "resources", "testProject", this)
 
 internal fun Path.addDefaultBuildFiles() {
     addPluginManagementToSettings()

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/diagnosticsRenderingSmoke/build.gradle.kts

@@ -0,0 +1,11 @@
+plugins {
+    kotlin("multiplatform").apply(false)
+}
+
+allprojects {
+    repositories {
+        mavenLocal()
+        maven("<localRepo>")
+        mavenCentral()
+    }
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/diagnosticsRenderingSmoke/expectedOutput.txt

@@ -0,0 +1,21 @@
+> Configure project :
+
+> Configure project :subprojectA
+w: [HierarchicalMultiplatformFlagsWarning | WARNING] The following properties are obsolete and will be removed in Kotlin 1.9.20:
+kotlin.mpp.enableCompatibilityMetadataVariant, kotlin.mpp.hierarchicalStructureSupport
+Read the details here: https://kotlinlang.org/docs/multiplatform-compatibility-guide.html#deprecate-hmpp-properties
+#diagnostic-end
+
+w: [CommonMainWithDependsOnDiagnostic | WARNING] commonMain can't declare dependsOn on other source sets
+#diagnostic-end
+
+
+> Configure project :subprojectB
+w: [CommonMainWithDependsOnDiagnostic | WARNING] commonMain can't declare dependsOn on other source sets
+#diagnostic-end
+
+w: [UnusedSourceSetsWarning | WARNING] The Kotlin source set unusedCreatedInAfterEvaluate was configured but not added to any Kotlin compilation.
+
+You can add a source set to a target's compilation by connecting it with the compilation's default source set using 'dependsOn'.
+See https://kotl.in/connecting-source-sets
+#diagnostic-end

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/diagnosticsRenderingSmoke/gradle.properties

@@ -0,0 +1,6 @@
+# Check that warnings from root project are reported
+kotlin.mpp.enableCompatibilityMetadataVariant=true
+
+# Check that warnings from root project are deduplicated with warnings from subprojects
+# (using small trick -- 'true' state is default but still provokes warnings)
+kotlin.mpp.hierarchicalStructureSupport=true

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/diagnosticsRenderingSmoke/settings.gradle.kts

@@ -0,0 +1,3 @@
+rootProject.name = "diagnostics-rendering-smoke"
+
+include("subprojectA", "subprojectB")

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/diagnosticsRenderingSmoke/subprojectA/build.gradle.kts

@@ -0,0 +1,24 @@
+plugins {
+    kotlin("multiplatform")
+}
+
+kotlin {
+    // targets do not matter just need Kotlin MPP Plugin
+    jvm()
+    linuxX64()
+
+    sourceSets {
+        val myCustomSourceSet by creating
+
+        // check that usual diagnostics are not deduplicated even if they are exactly the same
+        val commonMain by getting {
+            dependsOn(myCustomSourceSet)
+        }
+    }
+}
+
+afterEvaluate {
+    // NB: HMPP-flags are reported by the custom code that needs to run directly after plugin application rather than in
+    // afterEvaluate, so this reporting will be missed
+    setProperty("kotlin.native.enableDependencyPropagation", "false")
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/diagnosticsRenderingSmoke/subprojectA/gradle.properties

@@ -0,0 +1,3 @@
+# Check that warnings from root project are deduplicated with warnings from subprojects
+# (NB: default state provokes warning)
+kotlin.mpp.hierarchicalStructureSupport=true

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/diagnosticsRenderingSmoke/subprojectB/build.gradle.kts

@@ -0,0 +1,23 @@
+plugins {
+    kotlin("multiplatform")
+}
+
+kotlin {
+    // targets do not matter just need Kotlin MPP Plugin
+    jvm()
+    linuxX64()
+
+    sourceSets {
+        val myCustomSourceSet by creating
+
+        // check that usual diagnostics are not deduplicated even if they are exactly the same
+        val commonMain by getting {
+            dependsOn(myCustomSourceSet)
+        }
+
+        afterEvaluate {
+            // Check that changes made in trivial afterEvaluate are picked up
+            val unusedCreatedInAfterEvaluate by creating { }
+        }
+    }
+}

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/resources/testProject/diagnosticsRenderingSmoke/subprojectB/gradle.properties

@@ -0,0 +1,3 @@
+# Check that warnings from root project are deduplicated with warnings from subprojects
+# (NB: default state provokes warning)
+kotlin.mpp.hierarchicalStructureSupport=true