Add tests for statistic report after build finish

2022-07-05 20:02:40 +02:00
parent 2520ef9232
commit 3b84284743
4 changed files with 124 additions and 64 deletions
@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.gradle

 import com.google.gson.Gson
+import com.google.gson.JsonParser
 import io.ktor.application.*
 import io.ktor.http.*
 import io.ktor.request.*
@@ -15,28 +16,25 @@ import io.ktor.server.engine.*
 import io.ktor.server.netty.*
 import io.ktor.util.collections.*
 import org.gradle.util.GradleVersion
-import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData
-import org.jetbrains.kotlin.gradle.plugin.stat.StatTag
+import org.jetbrains.kotlin.gradle.plugin.stat.*
 import org.jetbrains.kotlin.gradle.report.BuildReportType
 import org.jetbrains.kotlin.gradle.testbase.*
+import org.jetbrains.kotlin.gradle.utils.`is`
 import org.jetbrains.kotlin.test.util.joinToArrayString
 import org.junit.jupiter.api.AfterAll
 import org.junit.jupiter.api.BeforeAll
 import org.junit.jupiter.api.DisplayName
 import java.io.IOException
-import java.net.HttpURLConnection
 import java.net.ServerSocket
-import java.net.URL
 import java.util.*
 import kotlin.io.path.appendText
-import kotlin.test.fail

@DisplayName("Build statistics")
@JvmGradlePluginTests
 class BuildStatisticsWithKtorIT : KGPBaseTest() {

    companion object {
-        fun CompileStatisticsData.validateMandatoryField(kotlinVersion: String, validationData: ValidationData): List<String> {
+        fun CompileStatisticsData.validateMandatoryField(kotlinVersion: String, validationData: TaskExecutionValidationData): List<String> {
            val validationErrors = LinkedList<String>()
            if (taskResult != validationData.taskResult) {
                validationErrors.add("Unexpected taskResult: $taskResult instead of ${validationData.taskResult}")
@@ -58,12 +56,10 @@ class BuildStatisticsWithKtorIT : KGPBaseTest() {
                    validationErrors.add("Does not contains \'$tag\' tag")
                }
            }
-
-
            return validationErrors
        }

-        fun CompileStatisticsData.validateIncrementalData(validationData: ValidationData): List<String> {
+        fun CompileStatisticsData.validateIncrementalData(validationData: TaskExecutionValidationData): List<String> {
            if (validationData.nonIncrementalReasons.isNotEmpty()) return emptyList()
            val validationErrors = LinkedList<String>()
            if (changes.size != validationData.changedFiles.size) {
@@ -75,7 +71,7 @@ class BuildStatisticsWithKtorIT : KGPBaseTest() {
            return validationErrors
        }

-        fun CompileStatisticsData.validateNonIncrementalData(validationData: ValidationData): List<String> {
+        fun CompileStatisticsData.validateNonIncrementalData(validationData: TaskExecutionValidationData): List<String> {
            if (validationData.nonIncrementalReasons.isEmpty()) return emptyList()
            val validationErrors = LinkedList<String>()
            if (!tags.contains(StatTag.NON_INCREMENTAL.name)) {
@@ -84,6 +80,15 @@ class BuildStatisticsWithKtorIT : KGPBaseTest() {
            return validationErrors
        }

+        fun BuildFinishStatisticsData.validate(validationData: BuildExecutionValidationData): List<String> {
+            if (validationData.tasks.isEmpty()) return emptyList()
+            val validationErrors = LinkedList<String>()
+            if (!startParameters.tasks.containsAll(validationData.tasks.toList())) {
+                validationErrors.add("Different set of executed tasks. Expected ${validationData.tasks.asList()} actual ${startParameters.tasks}")
+            }
+            return validationErrors
+        }
+
        fun getEmptyPort(): ServerSocket {
            for (port in 8080..8180) {
                try {
@@ -108,7 +113,7 @@ class BuildStatisticsWithKtorIT : KGPBaseTest() {
        println("start embedded server with data: ${ktorTestDataAnnotations.joinToArrayString()}")
        server = embeddedServer(Netty, port = port)
        {
-            val ktorServerData = ktorTestDataAnnotations.associateBy({ it.projectName }) { it.validationData.toMutableList() }
+            val ktorServerData = ktorTestDataAnnotations.associateBy { it.projectName }
            val failedResults = ConcurrentList<String>()

            suspend fun responseBadRequest(call: ApplicationCall, message: String) {
@@ -117,6 +122,62 @@ class BuildStatisticsWithKtorIT : KGPBaseTest() {
                call.respond(status = HttpStatusCode.BadRequest, message)
            }

+            suspend fun validateBuildExecutionData(
+                statisticsData: BuildFinishStatisticsData,
+                projectValidation: KtorTestData,
+                call: ApplicationCall
+            ) {
+                statisticsData.validate(projectValidation.buildValidationData).let {
+                    if (it.isNotEmpty()) {
+                        responseBadRequest(
+                            call,
+                            "${projectValidation.projectName}: Fail to validate statistics after build executed: ${it.joinToArrayString()}"
+                        )
+                        return
+                    }
+                    call.respond(HttpStatusCode.OK)
+                }
+            }
+
+            suspend fun validateTaskExecutionData(statData: CompileStatisticsData, projectValidation: KtorTestData, call: ApplicationCall) {
+                val ktorTestData = projectValidation.validationData.firstOrNull { it.taskName == statData.taskName }
+
+                if (ktorTestData == null) {
+                    responseBadRequest(call, "${projectValidation.projectName}: Unknown validation for task ${statData.taskName}")
+                    return
+                }
+
+                //validate response
+                statData.validateMandatoryField(defaultBuildOptions.kotlinVersion, ktorTestData).let {
+                    if (it.isNotEmpty()) {
+                        responseBadRequest(
+                            call,
+                            "${projectValidation.projectName}: Fail to validate mandatory fields: ${it.joinToArrayString()}"
+                        )
+                        return
+                    }
+                }
+                statData.validateIncrementalData(ktorTestData).let {
+                    if (it.isNotEmpty()) {
+                        responseBadRequest(
+                            call,
+                            "${projectValidation.projectName}: Fail to validate incremental fields: ${it.joinToArrayString()}"
+                        )
+                        return
+                    }
+                }
+                statData.validateNonIncrementalData(ktorTestData).let {
+                    if (it.isNotEmpty()) {
+                        responseBadRequest(
+                            call,
+                            "${projectValidation.projectName}: Fail to validate non-incremental fields: ${it.joinToArrayString()}"
+                        )
+                        return
+                    }
+                }
+                call.respond(HttpStatusCode.OK)
+            }
+
            routing {
                post("/badRequest") {
                    call.respond(HttpStatusCode.BadRequest, "Some reason")
@@ -125,49 +186,29 @@ class BuildStatisticsWithKtorIT : KGPBaseTest() {
                    val body = call.receive<String>()
                    println("TRACE: routing was called: $body")

-                    val statData = Gson().fromJson(body, CompileStatisticsData::class.java)
-                    val projectValidation = ktorServerData[statData.projectName]
+                    val jsonObject = JsonParser.parseString(body).asJsonObject
+                    val projectName = jsonObject["projectName"].asString
+
+                    val projectValidation = ktorServerData[projectName]
                    if (projectValidation == null) {
-                        responseBadRequest(call, "Unknown validation for project ${statData.projectName}")
+                        responseBadRequest(call, "Unknown validation for project $projectName")
                        return@post
                    }

-                    val ktorTestData = projectValidation.firstOrNull { it.taskName == statData.taskName }
+                    val type = jsonObject["type"].asString

-                    if (ktorTestData == null) {
-                        responseBadRequest(call, "${statData.projectName}: Unknown validation for task ${statData.taskName}")
-                        return@post
+                    when (BuildDataType.valueOf(type)) {
+                        BuildDataType.TASK_DATA -> validateTaskExecutionData(
+                            Gson().fromJson(body, CompileStatisticsData::class.java),
+                            projectValidation,
+                            call
+                        )
+                        BuildDataType.BUILD_DATA -> validateBuildExecutionData(
+                            Gson().fromJson(body, BuildFinishStatisticsData::class.java),
+                            projectValidation,
+                            call
+                        )
                    }
-
-                    //validate response
-                    statData.validateMandatoryField(defaultBuildOptions.kotlinVersion, ktorTestData).let {
-                        if (it.isNotEmpty()) {
-                            responseBadRequest(
-                                call,
-                                "${statData.projectName}: Fail to validate mandatory fields: ${it.joinToArrayString()}"
-                            )
-                            return@post
-                        }
-                    }
-                    statData.validateIncrementalData(ktorTestData).let {
-                        if (it.isNotEmpty()) {
-                            responseBadRequest(
-                                call,
-                                "${statData.projectName}: Fail to validate incremental fields: ${it.joinToArrayString()}"
-                            )
-                            return@post
-                        }
-                    }
-                    statData.validateNonIncrementalData(ktorTestData).let {
-                        if (it.isNotEmpty()) {
-                            responseBadRequest(
-                                call,
-                                "${statData.projectName}: Fail to validate non-incremental fields: ${it.joinToArrayString()}"
-                            )
-                            return@post
-                        }
-                    }
-                    call.respond(HttpStatusCode.OK)
                }
                get("/results") {
                    if (failedResults.isEmpty()) {
@@ -202,9 +243,10 @@ class BuildStatisticsWithKtorIT : KGPBaseTest() {
        "validateMandatoryField",
        [
            //first run
-            ValidationData(":lib:compileKotlin", expectedTags = ["NON_INCREMENTAL"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"]),
-            ValidationData(":app:compileKotlin", expectedTags = ["NON_INCREMENTAL"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"])
-        ]
+            TaskExecutionValidationData(":lib:compileKotlin", expectedTags = ["NON_INCREMENTAL"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"]),
+            TaskExecutionValidationData(":app:compileKotlin", expectedTags = ["NON_INCREMENTAL"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"])
+        ],
+        BuildExecutionValidationData(["assemble"])
    )
    @DisplayName("Validate mandatory field for http request body")
    @GradleTest
@@ -222,12 +264,13 @@ class BuildStatisticsWithKtorIT : KGPBaseTest() {
        "validateConfigurationCache",
        [
            //first run
-            ValidationData(":lib:compileKotlin", expectedTags = ["NON_INCREMENTAL", "CONFIGURATION_CACHE"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"]),
-            ValidationData(":app:compileKotlin", expectedTags = ["NON_INCREMENTAL", "CONFIGURATION_CACHE"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"]),
+            TaskExecutionValidationData(":lib:compileKotlin", expectedTags = ["NON_INCREMENTAL", "CONFIGURATION_CACHE"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"]),
+            TaskExecutionValidationData(":app:compileKotlin", expectedTags = ["NON_INCREMENTAL", "CONFIGURATION_CACHE"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"]),
            //second run
-            ValidationData(":lib:compileKotlin", expectedTags = ["INCREMENTAL", "CONFIGURATION_CACHE"]),
-            ValidationData(":app:compileKotlin", expectedTags = ["INCREMENTAL", "CONFIGURATION_CACHE"]),
-       ]
+            TaskExecutionValidationData(":lib:compileKotlin", expectedTags = ["INCREMENTAL", "CONFIGURATION_CACHE"]),
+            TaskExecutionValidationData(":app:compileKotlin", expectedTags = ["INCREMENTAL", "CONFIGURATION_CACHE"]),
+       ],
+        BuildExecutionValidationData(["assemble"])
    )
    @DisplayName("Validate configuration cache tag")
    @GradleTest
@@ -256,10 +299,11 @@ class BuildStatisticsWithKtorIT : KGPBaseTest() {
@Retention(AnnotationRetention.RUNTIME)
 annotation class KtorTestData(
    val projectName: String,
-    val validationData: Array<ValidationData>,
+    val validationData: Array<TaskExecutionValidationData>,
+    val buildValidationData: BuildExecutionValidationData = BuildExecutionValidationData()
 )

-annotation class ValidationData(
+annotation class TaskExecutionValidationData(
    //mandatory fields
    val taskName: String,
    val taskResult: String = "SUCCESS",
@@ -272,5 +316,12 @@ annotation class ValidationData(
    val changedFiles: Array<String> = [],
 )

+annotation class BuildExecutionValidationData(
+    val tasks: Array<String> = [],
+    val excludedTasks: Array<String> = [],
+    val projectProperties: Array<String> = [],
+    val systemProperties: Array<String> = [],
+)
+


@@ -30,7 +30,8 @@ data class CompileStatisticsData(
    val nonIncrementalAttributes: Set<BuildAttribute>,
    //TODO think about it,time in milliseconds
    val buildTimesMetrics: Map<BuildTime, Long>,
-    val performanceMetrics: Map<BuildPerformanceMetric, Long>
+    val performanceMetrics: Map<BuildPerformanceMetric, Long>,
+    val type: String = BuildDataType.TASK_DATA.name
 ) {
    companion object {
        private val formatter = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").also { it.timeZone = TimeZone.getTimeZone("UTC")}
@@ -48,6 +49,11 @@ enum class StatTag {
    BUILD_CACHE,
 }

+enum class BuildDataType {
+    TASK_DATA,
+    BUILD_DATA
+}
+
 //Sensitive data. This object is used directly for statistic via http
 data class GradleBuildStartParameters(
    val tasks: List<String>,
@@ -58,11 +64,13 @@ data class GradleBuildStartParameters(
 ) : java.io.Serializable

 //Sensitive data. This object is used directly for statistic via http
-data class BuildFinishData(
+data class BuildFinishStatisticsData(
+    val projectName: String,
    val startParameters: GradleBuildStartParameters,
    val buildUuid: String = "Unset",
    val label: String?,
    val totalTime: Long,
+    val type: String = BuildDataType.BUILD_DATA.name
 )


@@ -115,7 +115,7 @@ abstract class BuildMetricsReporterService : BuildService<BuildMetricsReporterSe

        fun getStartParameters(project: Project) = project.gradle.startParameter.let {
            GradleBuildStartParameters(
-                tasks = it.taskRequests.map { it.args.toString() },
+                tasks = it.taskRequests.flatMap { it.args },
                excludedTasks = it.excludedTaskNames,
                currentDir = it.currentDir.path,
                projectProperties = it.projectProperties.map {(key, value) -> "$key: $value"},
@@ -15,9 +15,8 @@ import org.gradle.api.services.BuildServiceParameters
 import org.gradle.tooling.events.FinishEvent
 import org.gradle.tooling.events.OperationCompletionListener
 import org.gradle.tooling.events.task.TaskFinishEvent
-import org.jetbrains.kotlin.gradle.plugin.stat.BuildFinishData
+import org.jetbrains.kotlin.gradle.plugin.stat.BuildFinishStatisticsData
 import org.jetbrains.kotlin.gradle.plugin.BuildEventsListenerRegistryHolder
-import org.jetbrains.kotlin.gradle.plugin.stat.CompileStatisticsData
 import org.jetbrains.kotlin.gradle.plugin.stat.GradleBuildStartParameters
 import org.jetbrains.kotlin.gradle.plugin.stat.StatTag
 import org.jetbrains.kotlin.gradle.plugin.statistics.BuildScanStatisticsListener
@@ -159,6 +158,7 @@ abstract class HttpReportService : BuildService<HttpReportService.Parameters>,
                checkResponseAndLog(connection)
                connection.inputStream.use { it.reader().use { reader -> reader.readText() } }
            } catch (e: Exception) {
+                log.debug("Unexpected exception happened ${e.message}: ${e.stackTrace}")
                checkResponseAndLog(connection)
            } finally {
                connection.disconnect()
@@ -168,7 +168,8 @@ abstract class HttpReportService : BuildService<HttpReportService.Parameters>,
    }

    private fun reportBuildFinish() {
-        val buildFinishData = BuildFinishData(
+        val buildFinishData = BuildFinishStatisticsData(
+            projectName = parameters.projectName,
            startParameters = parameters.startParameters,
            buildUuid = buildUuid,
            label = parameters.label,