From d6bbab7eff89e8c4a93d05c4e674ab88c86a8964 Mon Sep 17 00:00:00 2001 From: "nataliya.valtman" Date: Mon, 28 Nov 2022 18:35:36 +0100 Subject: [PATCH] Fix unstable build statistics with ktor tests --- .../gradle/BuildStatisticsWithKtorIT.kt | 400 +++++++----------- .../gradle/report/BuildReportsService.kt | 12 +- .../kotlin/gradle/report/HttpService.kt | 83 ++++ 3 files changed, 239 insertions(+), 256 deletions(-) create mode 100644 libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/HttpService.kt diff --git a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildStatisticsWithKtorIT.kt b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildStatisticsWithKtorIT.kt index 9a4bca2965b..7b2036df4d4 100644 --- a/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildStatisticsWithKtorIT.kt +++ b/libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/BuildStatisticsWithKtorIT.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.gradle import com.google.gson.Gson +import com.google.gson.JsonObject import com.google.gson.JsonParser import io.ktor.application.* import io.ktor.http.* @@ -19,80 +20,56 @@ import org.gradle.util.GradleVersion import org.jetbrains.kotlin.gradle.plugin.stat.* import org.jetbrains.kotlin.gradle.report.BuildReportType import org.jetbrains.kotlin.gradle.testbase.* -import org.jetbrains.kotlin.gradle.utils.`is` -import org.jetbrains.kotlin.test.util.joinToArrayString -import org.junit.jupiter.api.AfterAll -import org.junit.jupiter.api.BeforeAll +import org.jetbrains.kotlin.test.testFramework.KtUsefulTestCase.assertNotEmpty import org.junit.jupiter.api.DisplayName import java.io.IOException +import java.net.HttpURLConnection import java.net.ServerSocket +import java.net.URL import java.util.* -import kotlin.io.path.appendText +import java.util.concurrent.ArrayBlockingQueue +import java.util.concurrent.TimeUnit +import kotlin.test.assertContains +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFails +import kotlin.test.assertTrue @DisplayName("Build statistics") @JvmGradlePluginTests class BuildStatisticsWithKtorIT : KGPBaseTest() { companion object { - fun CompileStatisticsData.validateMandatoryField(kotlinVersion: String, validationData: TaskExecutionValidationData): List { - val validationErrors = LinkedList() - if (taskResult != validationData.taskResult) { - validationErrors.add("Unexpected taskResult: $taskResult instead of ${validationData.taskResult}") - } - if (this.kotlinVersion != kotlinVersion) { - validationErrors.add("Unexpected kotlinVersion: ${this.kotlinVersion} instead of ${kotlinVersion}") - } - if (compilerArguments.isEmpty()) { - validationErrors.add("Empty compiler arguments") - } - if (performanceMetrics.isEmpty()) { - validationErrors.add("Empty performance metrics") - } - if (buildTimesMetrics.isEmpty()) { - validationErrors.add("Empty build metrics") - } - for (tag: String in validationData.expectedTags) { - if (!tags.contains(tag)) { - validationErrors.add("Does not contains \'$tag\' tag") + fun runWithKtorService(action: (Int) -> Unit) { + val port = getEmptyPort().localPort + val server = embeddedServer(Netty, host="localhost", port = port) + { + val requests = ArrayBlockingQueue(10) + + routing { + post("/badRequest") { + call.respond(HttpStatusCode.BadRequest, "Some reason") + } + post("/put") { + val body = call.receive() + requests.add(body) + call.respond(HttpStatusCode.OK) + } + get("/validate") { + try { + call.respond(status = HttpStatusCode.OK, requests.poll(2, TimeUnit.SECONDS)) + } catch (e: Exception) { + call.respond(status = HttpStatusCode.NotFound, e.message ?: e::class) + } + } + } - } - return validationErrors + }.start() + action(port) + server.stop(1000, 1000) } - fun CompileStatisticsData.validateIncrementalData(validationData: TaskExecutionValidationData): List { - if (validationData.nonIncrementalReasons.isNotEmpty()) return emptyList() - val validationErrors = LinkedList() - if (changes.size != validationData.changedFiles.size) { - validationErrors.add("Changed files do not equal: ${changes.joinToArrayString()} instead of ${validationData.changedFiles.joinToArrayString()} ") - } - if (!tags.contains(StatTag.INCREMENTAL.name)) { - validationErrors.add("INCREMENTAL tag was no set") - } - return validationErrors - } - - fun CompileStatisticsData.validateNonIncrementalData(validationData: TaskExecutionValidationData): List { - if (validationData.nonIncrementalReasons.isEmpty()) return emptyList() - val validationErrors = LinkedList() - if (!tags.contains(StatTag.NON_INCREMENTAL.name)) { - validationErrors.add("NON_INCREMENTAL tag was no set") - } - return validationErrors - } - - fun BuildFinishStatisticsData.validate(validationData: BuildExecutionValidationData): List { - if (validationData.tasks.isEmpty()) return emptyList() - val validationErrors = LinkedList() - if (!startParameters.tasks.containsAll(validationData.tasks.toList())) { - validationErrors.add("Different set of executed tasks. Expected ${validationData.tasks.asList()} actual ${startParameters.tasks}") - } - if (gitBranch.isBlank()) { - validationErrors.add("Git branch is not set") - } - return validationErrors - } - - fun getEmptyPort(): ServerSocket { + private fun getEmptyPort(): ServerSocket { val startPort = 8080 val endPort = 8180 for (port in startPort..endPort) { @@ -104,232 +81,149 @@ class BuildStatisticsWithKtorIT : KGPBaseTest() { } catch (_: IOException) { continue // try next port } - } throw IOException("Failed to find free IP port in range $startPort..$endPort") } - } - - private lateinit var server: ApplicationEngine - - private val port = getEmptyPort().localPort - - @BeforeAll - fun initEmbeddedService() { - val ktorTestDataAnnotations = this.javaClass.methods.flatMap { it.annotations.filterIsInstance() } - println("start embedded server with data: ${ktorTestDataAnnotations.joinToArrayString()}") - server = embeddedServer(Netty, port = port) - { - val ktorServerData = ktorTestDataAnnotations.associateBy { it.projectName } - val failedResults = ConcurrentList() - - suspend fun responseBadRequest(call: ApplicationCall, message: String) { - println("Validation errors: $message") - failedResults.plus(message) - call.respond(status = HttpStatusCode.BadRequest, message) - } - - suspend fun validateBuildExecutionData( - statisticsData: BuildFinishStatisticsData, - projectValidation: KtorTestData, - call: ApplicationCall - ) { - statisticsData.validate(projectValidation.buildValidationData).let { - if (it.isNotEmpty()) { - responseBadRequest( - call, - "${projectValidation.projectName}: Fail to validate statistics after build executed: ${it.joinToArrayString()}" - ) - return - } - call.respond(HttpStatusCode.OK) - } - } - - suspend fun validateTaskExecutionData(statData: CompileStatisticsData, projectValidation: KtorTestData, call: ApplicationCall) { - val ktorTestData = projectValidation.validationData.firstOrNull { it.taskName == statData.taskName } - - if (ktorTestData == null) { - responseBadRequest(call, "${projectValidation.projectName}: Unknown validation for task ${statData.taskName}") - return - } - - //validate response - statData.validateMandatoryField(defaultBuildOptions.kotlinVersion, ktorTestData).let { - if (it.isNotEmpty()) { - responseBadRequest( - call, - "${projectValidation.projectName}: Fail to validate mandatory fields: ${it.joinToArrayString()}" - ) - return - } - } - statData.validateIncrementalData(ktorTestData).let { - if (it.isNotEmpty()) { - responseBadRequest( - call, - "${projectValidation.projectName}: Fail to validate incremental fields: ${it.joinToArrayString()}" - ) - return - } - } - statData.validateNonIncrementalData(ktorTestData).let { - if (it.isNotEmpty()) { - responseBadRequest( - call, - "${projectValidation.projectName}: Fail to validate non-incremental fields: ${it.joinToArrayString()}" - ) - return - } - } - call.respond(HttpStatusCode.OK) - } - - routing { - post("/badRequest") { - call.respond(HttpStatusCode.BadRequest, "Some reason") - } - post("/validate") { - val body = call.receive() - println("TRACE: routing was called: $body") - - val jsonObject = JsonParser.parseString(body).asJsonObject - val projectName = jsonObject["projectName"].asString - - val projectValidation = ktorServerData[projectName] - if (projectValidation == null) { - responseBadRequest(call, "Unknown validation for project $projectName") - return@post - } - - val type = jsonObject["type"].asString - - when (BuildDataType.valueOf(type)) { - BuildDataType.TASK_DATA -> validateTaskExecutionData( - Gson().fromJson(body, CompileStatisticsData::class.java), - projectValidation, - call - ) - BuildDataType.BUILD_DATA -> validateBuildExecutionData( - Gson().fromJson(body, BuildFinishStatisticsData::class.java), - projectValidation, - call - ) - } - } - get("/results") { - if (failedResults.isEmpty()) { - call.respond(HttpStatusCode.OK) - } else { - call.respond(HttpStatusCode.InternalServerError, message = failedResults.joinToArrayString()) - } - } + private fun validateCall(port: Int, validate: (JsonObject) -> Unit) { + try { + val connection = URL("http://localhost:$port/validate").openConnection() as HttpURLConnection + connection.requestMethod = "GET" + connection.connect() + assertEquals(HttpStatusCode.OK.value, connection.responseCode) + val body = connection.inputStream.bufferedReader().readText() + val jsonObject = JsonParser.parseString(body).asJsonObject + validate(jsonObject) + } catch (e: IOException) { + assertFails { "Unable to open connection : ${e.message}" } } } - server.start() - } - @AfterAll - fun shutDownEmbeddedService() { - server.stop(1000, 1000) - } + fun validateTaskData(port: Int, validate: (CompileStatisticsData) -> Unit) { + validateCall(port) { jsonObject -> + val type = jsonObject["type"].asString + assertEquals(BuildDataType.TASK_DATA, BuildDataType.valueOf(type)) + val taskData = Gson().fromJson(jsonObject, CompileStatisticsData::class.java) + validate(taskData) + } + } + fun validateBuildData(port: Int, validate: (BuildFinishStatisticsData) -> Unit) { + validateCall(port) { jsonObject -> + val type = jsonObject["type"].asString + assertEquals(BuildDataType.BUILD_DATA, BuildDataType.valueOf(type)) + val buildData = Gson().fromJson(jsonObject, BuildFinishStatisticsData::class.java) + validate(buildData) + } + } + + } @DisplayName("Http build report request problems are logged only ones") @GradleTest fun testHttpServiceWithBadRequest(gradleVersion: GradleVersion) { - project("incrementalMultiproject", gradleVersion) { - enableStatisticReports(BuildReportType.HTTP, "http://localhost:$port/badRequest") - build("assemble") { - assertOutputContainsExactTimes("Failed to send statistic to", 1) + runWithKtorService { port -> + project("incrementalMultiproject", gradleVersion) { + enableStatisticReports(BuildReportType.HTTP, "http://localhost:$port/badRequest") + build("assemble") { + assertOutputContainsExactTimes("Failed to send statistic to", 1) + } } } } - @KtorTestData( - "validateMandatoryField", - [ - //first run - TaskExecutionValidationData(":lib:compileKotlin", expectedTags = ["NON_INCREMENTAL"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"]), - TaskExecutionValidationData(":app:compileKotlin", expectedTags = ["NON_INCREMENTAL"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"]) - ], - BuildExecutionValidationData(["assemble"]) - ) @DisplayName("Validate mandatory field for http request body") @GradleTest fun testHttpRequest(gradleVersion: GradleVersion) { - project("incrementalMultiproject", gradleVersion) { - setProjectForTest("validateMandatoryField") - build("assemble") { - assertOutputDoesNotContain("Failed to send statistic to") + runWithKtorService { port -> + project("incrementalMultiproject", gradleVersion) { + setProjectForTest(port) + build("clean", "assemble") { + assertOutputDoesNotContain("Failed to send statistic to") + } + } + validateTaskData(port) { taskData -> + assertEquals(":lib:compileKotlin", taskData.taskName) + assertContains(taskData.tags, "NON_INCREMENTAL") + assertContains(taskData.nonIncrementalAttributes.map { it.name }, "UNKNOWN_CHANGES_IN_GRADLE_INPUTS") + assertTrue(taskData.performanceMetrics.keys.isNotEmpty()) + assertTrue(taskData.buildTimesMetrics.keys.isNotEmpty()) + assertEquals( + defaultBuildOptions.kotlinVersion, taskData.kotlinVersion, + "Unexpected kotlinVersion: ${taskData.kotlinVersion} instead of ${defaultBuildOptions.kotlinVersion}" + ) + } + validateTaskData(port) { taskData -> + assertEquals(":app:compileKotlin", taskData.taskName) + assertContains(taskData.tags, "NON_INCREMENTAL") + assertContains(taskData.nonIncrementalAttributes.map { it.name }, "UNKNOWN_CHANGES_IN_GRADLE_INPUTS") + assertTrue(taskData.performanceMetrics.keys.isNotEmpty()) + assertTrue(taskData.buildTimesMetrics.keys.isNotEmpty()) + assertEquals( + defaultBuildOptions.kotlinVersion, taskData.kotlinVersion, + "Unexpected kotlinVersion: ${taskData.kotlinVersion} instead of ${defaultBuildOptions.kotlinVersion}" + ) + } + validateBuildData(port) { buildData -> + assertContains(buildData.startParameters.tasks, "assemble") } - } } - @KtorTestData( - "validateConfigurationCache", - [ - //first run - TaskExecutionValidationData(":lib:compileKotlin", expectedTags = ["NON_INCREMENTAL", "CONFIGURATION_CACHE"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"]), - TaskExecutionValidationData(":app:compileKotlin", expectedTags = ["NON_INCREMENTAL", "CONFIGURATION_CACHE"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"]), - //second run - TaskExecutionValidationData(":lib:compileKotlin", expectedTags = ["INCREMENTAL", "CONFIGURATION_CACHE"]), - TaskExecutionValidationData(":app:compileKotlin", expectedTags = ["INCREMENTAL", "CONFIGURATION_CACHE"]), - ], - BuildExecutionValidationData(["assemble"]) - ) @DisplayName("Validate configuration cache tag") @GradleTest fun testConfigurationCache(gradleVersion: GradleVersion) { - val buildOptions = defaultBuildOptions.copy(configurationCache = true) - project("incrementalMultiproject", gradleVersion) { - setProjectForTest("validateConfigurationCache") - build("assemble", buildOptions = buildOptions) { - assertOutputDoesNotContain("Failed to send statistic to") + runWithKtorService { port -> + + val buildOptions = defaultBuildOptions.copy(configurationCache = true) + project("incrementalMultiproject", gradleVersion) { + setProjectForTest(port) + build("assemble", buildOptions = buildOptions) { + assertOutputDoesNotContain("Failed to send statistic to") + } + projectPath.resolve("lib/src/main/kotlin/bar/B.kt").modify { it.replace("fun b() {}", "fun b() = 1") } + build("assemble", buildOptions = buildOptions) { + assertOutputDoesNotContain("Failed to send statistic to") + } } - projectPath.resolve("lib/src/main/kotlin/bar/B.kt") - build("assemble", buildOptions = buildOptions) { - assertOutputDoesNotContain("Failed to send statistic to") + validateTaskData(port) { taskData -> + assertEquals(":lib:compileKotlin", taskData.taskName) + assertContentEquals(listOf("ARTIFACT_TRANSFORM", "CONFIGURATION_CACHE", "NON_INCREMENTAL"), taskData.tags.sorted(), ) + assertEquals( + defaultBuildOptions.kotlinVersion, taskData.kotlinVersion, + "Unexpected kotlinVersion: ${taskData.kotlinVersion} instead of ${defaultBuildOptions.kotlinVersion}" + ) + } + validateTaskData(port) { taskData -> + assertEquals(":app:compileKotlin", taskData.taskName) + assertContentEquals(taskData.tags.sorted(), listOf("ARTIFACT_TRANSFORM", "CONFIGURATION_CACHE", "NON_INCREMENTAL")) + assertEquals( + defaultBuildOptions.kotlinVersion, taskData.kotlinVersion, + "Unexpected kotlinVersion: ${taskData.kotlinVersion} instead of ${defaultBuildOptions.kotlinVersion}" + ) + } + validateBuildData(port) { buildData -> + assertContains(buildData.startParameters.tasks, "assemble") + } + //second build + validateTaskData(port) { taskData -> + assertEquals(":lib:compileKotlin", taskData.taskName) + assertContentEquals(taskData.tags.sorted(), listOf("ARTIFACT_TRANSFORM", "CONFIGURATION_CACHE", "INCREMENTAL")) + } + validateTaskData(port) { taskData -> + assertEquals(":app:compileKotlin", taskData.taskName) + assertContentEquals(taskData.tags.sorted(), listOf("ARTIFACT_TRANSFORM", "CONFIGURATION_CACHE", "INCREMENTAL")) } } } - private fun TestProject.setProjectForTest(projectName: String) { - enableStatisticReports(BuildReportType.HTTP, "http://localhost:$port/validate") - settingsGradle.appendText("rootProject.name=\'$projectName\'") + private fun TestProject.setProjectForTest(port: Int) { + enableStatisticReports(BuildReportType.HTTP, "http://localhost:$port/put") } } -@Repeatable -@Target(AnnotationTarget.FUNCTION) -@Retention(AnnotationRetention.RUNTIME) -annotation class KtorTestData( - val projectName: String, - val validationData: Array, - val buildValidationData: BuildExecutionValidationData = BuildExecutionValidationData() -) - -annotation class TaskExecutionValidationData( - //mandatory fields - val taskName: String, - val taskResult: String = "SUCCESS", - val expectedTags: Array = [], - - //fields for non-incremental compilation - val nonIncrementalReasons: Array = [], //if empty incremental validation expected - - //fields for incremental compilation - val changedFiles: Array = [], -) - -annotation class BuildExecutionValidationData( - val tasks: Array = [], - val excludedTasks: Array = [], - val projectProperties: Array = [], - val systemProperties: Array = [], -) + + diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt index 9427a34a5b5..e0d07d968e4 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/BuildReportsService.kt @@ -56,6 +56,7 @@ abstract class BuildReportsService : BuildService val reportingSettings: Property var buildMetricsService: Provider + val httpService: Property val projectDir: DirectoryProperty val label: Property @@ -127,7 +128,7 @@ abstract class BuildReportsService : BuildService if (event is TaskFinishEvent) { val data = prepareData( @@ -157,7 +158,11 @@ abstract class BuildReportsService : BuildService it.parameters.httpService.set(HttpReportServiceImpl(httpSettings)) } it.parameters.buildMetricsService = buildMetricsService it.parameters.projectDir.set(project.rootProject.layout.projectDirectory) diff --git a/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/HttpService.kt b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/HttpService.kt new file mode 100644 index 00000000000..1510a3a0a79 --- /dev/null +++ b/libraries/tools/kotlin-gradle-plugin/src/common/kotlin/org/jetbrains/kotlin/gradle/report/HttpService.kt @@ -0,0 +1,83 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.gradle.report + +import com.google.gson.Gson +import org.gradle.api.logging.Logger +import java.io.IOException +import java.io.Serializable +import java.net.HttpURLConnection +import java.net.URL +import java.util.* +import kotlin.system.measureTimeMillis + +interface HttpReportService { + fun sendData(data: Any, log: Logger) +} + +class HttpReportServiceImpl( + private val url: String, + private val password: String?, + private val user: String?, +) : HttpReportService, Serializable { + constructor(httpSettings: HttpReportSettings) : this(httpSettings.url, httpSettings.password, httpSettings.user) + + companion object { + private var invalidUrl = false + private var requestPreviousFailed = false + + private fun checkResponseAndLog(connection: HttpURLConnection, log: Logger) { + val isResponseBad = connection.responseCode !in 200..299 + if (isResponseBad) { + val message = "Failed to send statistic to ${connection.url} with ${connection.responseCode}: ${connection.responseMessage}" + if (!requestPreviousFailed) { + log.warn(message) + } else { + log.debug(message) + } + requestPreviousFailed = true + } + } + } + + override fun sendData(data: Any, log: Logger) { + val elapsedTime = measureTimeMillis { + if (invalidUrl) { + return + } + val connection = try { + URL(url).openConnection() as HttpURLConnection + } catch (e: IOException) { + log.warn("Unable to open connection to ${url}: ${e.message}") + invalidUrl = true + return + } + + try { + if (user != null && password != null) { + val auth = Base64.getEncoder() + .encode("${user}:${password}".toByteArray()) + .toString(Charsets.UTF_8) + connection.addRequestProperty("Authorization", "Basic $auth") + } + connection.addRequestProperty("Content-Type", "application/json") + connection.requestMethod = "POST" + connection.doOutput = true + connection.outputStream.use { + it.write(Gson().toJson(data).toByteArray()) + } + connection.connect() + checkResponseAndLog(connection, log) + } catch (e: Exception) { + log.debug("Unexpected exception happened ${e.message}: ${e.stackTrace}") + checkResponseAndLog(connection, log) + } finally { + connection.disconnect() + } + } + log.debug("Report statistic by http takes $elapsedTime ms") + } +}