Fix unstable build statistics with ktor tests

This commit is contained in:
nataliya.valtman
2022-11-28 18:35:36 +01:00
parent e2d96da396
commit d6bbab7eff
3 changed files with 239 additions and 256 deletions
@@ -6,6 +6,7 @@
package org.jetbrains.kotlin.gradle
import com.google.gson.Gson
import com.google.gson.JsonObject
import com.google.gson.JsonParser
import io.ktor.application.*
import io.ktor.http.*
@@ -19,80 +20,56 @@ import org.gradle.util.GradleVersion
import org.jetbrains.kotlin.gradle.plugin.stat.*
import org.jetbrains.kotlin.gradle.report.BuildReportType
import org.jetbrains.kotlin.gradle.testbase.*
import org.jetbrains.kotlin.gradle.utils.`is`
import org.jetbrains.kotlin.test.util.joinToArrayString
import org.junit.jupiter.api.AfterAll
import org.junit.jupiter.api.BeforeAll
import org.jetbrains.kotlin.test.testFramework.KtUsefulTestCase.assertNotEmpty
import org.junit.jupiter.api.DisplayName
import java.io.IOException
import java.net.HttpURLConnection
import java.net.ServerSocket
import java.net.URL
import java.util.*
import kotlin.io.path.appendText
import java.util.concurrent.ArrayBlockingQueue
import java.util.concurrent.TimeUnit
import kotlin.test.assertContains
import kotlin.test.assertContentEquals
import kotlin.test.assertEquals
import kotlin.test.assertFails
import kotlin.test.assertTrue
@DisplayName("Build statistics")
@JvmGradlePluginTests
class BuildStatisticsWithKtorIT : KGPBaseTest() {
companion object {
fun CompileStatisticsData.validateMandatoryField(kotlinVersion: String, validationData: TaskExecutionValidationData): List<String> {
val validationErrors = LinkedList<String>()
if (taskResult != validationData.taskResult) {
validationErrors.add("Unexpected taskResult: $taskResult instead of ${validationData.taskResult}")
}
if (this.kotlinVersion != kotlinVersion) {
validationErrors.add("Unexpected kotlinVersion: ${this.kotlinVersion} instead of ${kotlinVersion}")
}
if (compilerArguments.isEmpty()) {
validationErrors.add("Empty compiler arguments")
}
if (performanceMetrics.isEmpty()) {
validationErrors.add("Empty performance metrics")
}
if (buildTimesMetrics.isEmpty()) {
validationErrors.add("Empty build metrics")
}
for (tag: String in validationData.expectedTags) {
if (!tags.contains(tag)) {
validationErrors.add("Does not contains \'$tag\' tag")
fun runWithKtorService(action: (Int) -> Unit) {
val port = getEmptyPort().localPort
val server = embeddedServer(Netty, host="localhost", port = port)
{
val requests = ArrayBlockingQueue<String>(10)
routing {
post("/badRequest") {
call.respond(HttpStatusCode.BadRequest, "Some reason")
}
post("/put") {
val body = call.receive<String>()
requests.add(body)
call.respond(HttpStatusCode.OK)
}
get("/validate") {
try {
call.respond(status = HttpStatusCode.OK, requests.poll(2, TimeUnit.SECONDS))
} catch (e: Exception) {
call.respond(status = HttpStatusCode.NotFound, e.message ?: e::class)
}
}
}
}
return validationErrors
}.start()
action(port)
server.stop(1000, 1000)
}
fun CompileStatisticsData.validateIncrementalData(validationData: TaskExecutionValidationData): List<String> {
if (validationData.nonIncrementalReasons.isNotEmpty()) return emptyList()
val validationErrors = LinkedList<String>()
if (changes.size != validationData.changedFiles.size) {
validationErrors.add("Changed files do not equal: ${changes.joinToArrayString()} instead of ${validationData.changedFiles.joinToArrayString()} ")
}
if (!tags.contains(StatTag.INCREMENTAL.name)) {
validationErrors.add("INCREMENTAL tag was no set")
}
return validationErrors
}
fun CompileStatisticsData.validateNonIncrementalData(validationData: TaskExecutionValidationData): List<String> {
if (validationData.nonIncrementalReasons.isEmpty()) return emptyList()
val validationErrors = LinkedList<String>()
if (!tags.contains(StatTag.NON_INCREMENTAL.name)) {
validationErrors.add("NON_INCREMENTAL tag was no set")
}
return validationErrors
}
fun BuildFinishStatisticsData.validate(validationData: BuildExecutionValidationData): List<String> {
if (validationData.tasks.isEmpty()) return emptyList()
val validationErrors = LinkedList<String>()
if (!startParameters.tasks.containsAll(validationData.tasks.toList())) {
validationErrors.add("Different set of executed tasks. Expected ${validationData.tasks.asList()} actual ${startParameters.tasks}")
}
if (gitBranch.isBlank()) {
validationErrors.add("Git branch is not set")
}
return validationErrors
}
fun getEmptyPort(): ServerSocket {
private fun getEmptyPort(): ServerSocket {
val startPort = 8080
val endPort = 8180
for (port in startPort..endPort) {
@@ -104,232 +81,149 @@ class BuildStatisticsWithKtorIT : KGPBaseTest() {
} catch (_: IOException) {
continue // try next port
}
}
throw IOException("Failed to find free IP port in range $startPort..$endPort")
}
}
private lateinit var server: ApplicationEngine
private val port = getEmptyPort().localPort
@BeforeAll
fun initEmbeddedService() {
val ktorTestDataAnnotations = this.javaClass.methods.flatMap { it.annotations.filterIsInstance<KtorTestData>() }
println("start embedded server with data: ${ktorTestDataAnnotations.joinToArrayString()}")
server = embeddedServer(Netty, port = port)
{
val ktorServerData = ktorTestDataAnnotations.associateBy { it.projectName }
val failedResults = ConcurrentList<String>()
suspend fun responseBadRequest(call: ApplicationCall, message: String) {
println("Validation errors: $message")
failedResults.plus(message)
call.respond(status = HttpStatusCode.BadRequest, message)
}
suspend fun validateBuildExecutionData(
statisticsData: BuildFinishStatisticsData,
projectValidation: KtorTestData,
call: ApplicationCall
) {
statisticsData.validate(projectValidation.buildValidationData).let {
if (it.isNotEmpty()) {
responseBadRequest(
call,
"${projectValidation.projectName}: Fail to validate statistics after build executed: ${it.joinToArrayString()}"
)
return
}
call.respond(HttpStatusCode.OK)
}
}
suspend fun validateTaskExecutionData(statData: CompileStatisticsData, projectValidation: KtorTestData, call: ApplicationCall) {
val ktorTestData = projectValidation.validationData.firstOrNull { it.taskName == statData.taskName }
if (ktorTestData == null) {
responseBadRequest(call, "${projectValidation.projectName}: Unknown validation for task ${statData.taskName}")
return
}
//validate response
statData.validateMandatoryField(defaultBuildOptions.kotlinVersion, ktorTestData).let {
if (it.isNotEmpty()) {
responseBadRequest(
call,
"${projectValidation.projectName}: Fail to validate mandatory fields: ${it.joinToArrayString()}"
)
return
}
}
statData.validateIncrementalData(ktorTestData).let {
if (it.isNotEmpty()) {
responseBadRequest(
call,
"${projectValidation.projectName}: Fail to validate incremental fields: ${it.joinToArrayString()}"
)
return
}
}
statData.validateNonIncrementalData(ktorTestData).let {
if (it.isNotEmpty()) {
responseBadRequest(
call,
"${projectValidation.projectName}: Fail to validate non-incremental fields: ${it.joinToArrayString()}"
)
return
}
}
call.respond(HttpStatusCode.OK)
}
routing {
post("/badRequest") {
call.respond(HttpStatusCode.BadRequest, "Some reason")
}
post("/validate") {
val body = call.receive<String>()
println("TRACE: routing was called: $body")
val jsonObject = JsonParser.parseString(body).asJsonObject
val projectName = jsonObject["projectName"].asString
val projectValidation = ktorServerData[projectName]
if (projectValidation == null) {
responseBadRequest(call, "Unknown validation for project $projectName")
return@post
}
val type = jsonObject["type"].asString
when (BuildDataType.valueOf(type)) {
BuildDataType.TASK_DATA -> validateTaskExecutionData(
Gson().fromJson(body, CompileStatisticsData::class.java),
projectValidation,
call
)
BuildDataType.BUILD_DATA -> validateBuildExecutionData(
Gson().fromJson(body, BuildFinishStatisticsData::class.java),
projectValidation,
call
)
}
}
get("/results") {
if (failedResults.isEmpty()) {
call.respond(HttpStatusCode.OK)
} else {
call.respond(HttpStatusCode.InternalServerError, message = failedResults.joinToArrayString())
}
}
private fun validateCall(port: Int, validate: (JsonObject) -> Unit) {
try {
val connection = URL("http://localhost:$port/validate").openConnection() as HttpURLConnection
connection.requestMethod = "GET"
connection.connect()
assertEquals(HttpStatusCode.OK.value, connection.responseCode)
val body = connection.inputStream.bufferedReader().readText()
val jsonObject = JsonParser.parseString(body).asJsonObject
validate(jsonObject)
} catch (e: IOException) {
assertFails { "Unable to open connection : ${e.message}" }
}
}
server.start()
}
@AfterAll
fun shutDownEmbeddedService() {
server.stop(1000, 1000)
}
fun validateTaskData(port: Int, validate: (CompileStatisticsData) -> Unit) {
validateCall(port) { jsonObject ->
val type = jsonObject["type"].asString
assertEquals(BuildDataType.TASK_DATA, BuildDataType.valueOf(type))
val taskData = Gson().fromJson(jsonObject, CompileStatisticsData::class.java)
validate(taskData)
}
}
fun validateBuildData(port: Int, validate: (BuildFinishStatisticsData) -> Unit) {
validateCall(port) { jsonObject ->
val type = jsonObject["type"].asString
assertEquals(BuildDataType.BUILD_DATA, BuildDataType.valueOf(type))
val buildData = Gson().fromJson(jsonObject, BuildFinishStatisticsData::class.java)
validate(buildData)
}
}
}
@DisplayName("Http build report request problems are logged only ones")
@GradleTest
fun testHttpServiceWithBadRequest(gradleVersion: GradleVersion) {
project("incrementalMultiproject", gradleVersion) {
enableStatisticReports(BuildReportType.HTTP, "http://localhost:$port/badRequest")
build("assemble") {
assertOutputContainsExactTimes("Failed to send statistic to", 1)
runWithKtorService { port ->
project("incrementalMultiproject", gradleVersion) {
enableStatisticReports(BuildReportType.HTTP, "http://localhost:$port/badRequest")
build("assemble") {
assertOutputContainsExactTimes("Failed to send statistic to", 1)
}
}
}
}
@KtorTestData(
"validateMandatoryField",
[
//first run
TaskExecutionValidationData(":lib:compileKotlin", expectedTags = ["NON_INCREMENTAL"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"]),
TaskExecutionValidationData(":app:compileKotlin", expectedTags = ["NON_INCREMENTAL"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"])
],
BuildExecutionValidationData(["assemble"])
)
@DisplayName("Validate mandatory field for http request body")
@GradleTest
fun testHttpRequest(gradleVersion: GradleVersion) {
project("incrementalMultiproject", gradleVersion) {
setProjectForTest("validateMandatoryField")
build("assemble") {
assertOutputDoesNotContain("Failed to send statistic to")
runWithKtorService { port ->
project("incrementalMultiproject", gradleVersion) {
setProjectForTest(port)
build("clean", "assemble") {
assertOutputDoesNotContain("Failed to send statistic to")
}
}
validateTaskData(port) { taskData ->
assertEquals(":lib:compileKotlin", taskData.taskName)
assertContains(taskData.tags, "NON_INCREMENTAL")
assertContains(taskData.nonIncrementalAttributes.map { it.name }, "UNKNOWN_CHANGES_IN_GRADLE_INPUTS")
assertTrue(taskData.performanceMetrics.keys.isNotEmpty())
assertTrue(taskData.buildTimesMetrics.keys.isNotEmpty())
assertEquals(
defaultBuildOptions.kotlinVersion, taskData.kotlinVersion,
"Unexpected kotlinVersion: ${taskData.kotlinVersion} instead of ${defaultBuildOptions.kotlinVersion}"
)
}
validateTaskData(port) { taskData ->
assertEquals(":app:compileKotlin", taskData.taskName)
assertContains(taskData.tags, "NON_INCREMENTAL")
assertContains(taskData.nonIncrementalAttributes.map { it.name }, "UNKNOWN_CHANGES_IN_GRADLE_INPUTS")
assertTrue(taskData.performanceMetrics.keys.isNotEmpty())
assertTrue(taskData.buildTimesMetrics.keys.isNotEmpty())
assertEquals(
defaultBuildOptions.kotlinVersion, taskData.kotlinVersion,
"Unexpected kotlinVersion: ${taskData.kotlinVersion} instead of ${defaultBuildOptions.kotlinVersion}"
)
}
validateBuildData(port) { buildData ->
assertContains(buildData.startParameters.tasks, "assemble")
}
}
}
@KtorTestData(
"validateConfigurationCache",
[
//first run
TaskExecutionValidationData(":lib:compileKotlin", expectedTags = ["NON_INCREMENTAL", "CONFIGURATION_CACHE"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"]),
TaskExecutionValidationData(":app:compileKotlin", expectedTags = ["NON_INCREMENTAL", "CONFIGURATION_CACHE"], nonIncrementalReasons = ["UNKNOWN_CHANGES_IN_GRADLE_INPUTS"]),
//second run
TaskExecutionValidationData(":lib:compileKotlin", expectedTags = ["INCREMENTAL", "CONFIGURATION_CACHE"]),
TaskExecutionValidationData(":app:compileKotlin", expectedTags = ["INCREMENTAL", "CONFIGURATION_CACHE"]),
],
BuildExecutionValidationData(["assemble"])
)
@DisplayName("Validate configuration cache tag")
@GradleTest
fun testConfigurationCache(gradleVersion: GradleVersion) {
val buildOptions = defaultBuildOptions.copy(configurationCache = true)
project("incrementalMultiproject", gradleVersion) {
setProjectForTest("validateConfigurationCache")
build("assemble", buildOptions = buildOptions) {
assertOutputDoesNotContain("Failed to send statistic to")
runWithKtorService { port ->
val buildOptions = defaultBuildOptions.copy(configurationCache = true)
project("incrementalMultiproject", gradleVersion) {
setProjectForTest(port)
build("assemble", buildOptions = buildOptions) {
assertOutputDoesNotContain("Failed to send statistic to")
}
projectPath.resolve("lib/src/main/kotlin/bar/B.kt").modify { it.replace("fun b() {}", "fun b() = 1") }
build("assemble", buildOptions = buildOptions) {
assertOutputDoesNotContain("Failed to send statistic to")
}
}
projectPath.resolve("lib/src/main/kotlin/bar/B.kt")
build("assemble", buildOptions = buildOptions) {
assertOutputDoesNotContain("Failed to send statistic to")
validateTaskData(port) { taskData ->
assertEquals(":lib:compileKotlin", taskData.taskName)
assertContentEquals(listOf("ARTIFACT_TRANSFORM", "CONFIGURATION_CACHE", "NON_INCREMENTAL"), taskData.tags.sorted(), )
assertEquals(
defaultBuildOptions.kotlinVersion, taskData.kotlinVersion,
"Unexpected kotlinVersion: ${taskData.kotlinVersion} instead of ${defaultBuildOptions.kotlinVersion}"
)
}
validateTaskData(port) { taskData ->
assertEquals(":app:compileKotlin", taskData.taskName)
assertContentEquals(taskData.tags.sorted(), listOf("ARTIFACT_TRANSFORM", "CONFIGURATION_CACHE", "NON_INCREMENTAL"))
assertEquals(
defaultBuildOptions.kotlinVersion, taskData.kotlinVersion,
"Unexpected kotlinVersion: ${taskData.kotlinVersion} instead of ${defaultBuildOptions.kotlinVersion}"
)
}
validateBuildData(port) { buildData ->
assertContains(buildData.startParameters.tasks, "assemble")
}
//second build
validateTaskData(port) { taskData ->
assertEquals(":lib:compileKotlin", taskData.taskName)
assertContentEquals(taskData.tags.sorted(), listOf("ARTIFACT_TRANSFORM", "CONFIGURATION_CACHE", "INCREMENTAL"))
}
validateTaskData(port) { taskData ->
assertEquals(":app:compileKotlin", taskData.taskName)
assertContentEquals(taskData.tags.sorted(), listOf("ARTIFACT_TRANSFORM", "CONFIGURATION_CACHE", "INCREMENTAL"))
}
}
}
private fun TestProject.setProjectForTest(projectName: String) {
enableStatisticReports(BuildReportType.HTTP, "http://localhost:$port/validate")
settingsGradle.appendText("rootProject.name=\'$projectName\'")
private fun TestProject.setProjectForTest(port: Int) {
enableStatisticReports(BuildReportType.HTTP, "http://localhost:$port/put")
}
}
@Repeatable
@Target(AnnotationTarget.FUNCTION)
@Retention(AnnotationRetention.RUNTIME)
annotation class KtorTestData(
val projectName: String,
val validationData: Array<TaskExecutionValidationData>,
val buildValidationData: BuildExecutionValidationData = BuildExecutionValidationData()
)
annotation class TaskExecutionValidationData(
//mandatory fields
val taskName: String,
val taskResult: String = "SUCCESS",
val expectedTags: Array<String> = [],
//fields for non-incremental compilation
val nonIncrementalReasons: Array<String> = [], //if empty incremental validation expected
//fields for incremental compilation
val changedFiles: Array<String> = [],
)
annotation class BuildExecutionValidationData(
val tasks: Array<String> = [],
val excludedTasks: Array<String> = [],
val projectProperties: Array<String> = [],
val systemProperties: Array<String> = [],
)
@@ -56,6 +56,7 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
val startParameters: Property<GradleBuildStartParameters>
val reportingSettings: Property<ReportingSettings>
var buildMetricsService: Provider<BuildMetricsService>
val httpService: Property<HttpReportService>
val projectDir: DirectoryProperty
val label: Property<String?>
@@ -127,7 +128,7 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
gitBranch = branchName
)
sendDataViaHttp(httpReportSettings, buildFinishData, log)
parameters.httpService.orNull?.sendData(buildFinishData, log)
}
private fun GradleBuildStartParameters.includeVerboseEnvironment(verboseEnvironment: Boolean): GradleBuildStartParameters {
@@ -145,7 +146,7 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
}
private fun addHttpReport(event: FinishEvent?) {
if (parameters.reportingSettings.get().httpReportSettings != null) {
parameters.httpService.orNull?.also { httpService ->
if (event is TaskFinishEvent) {
val data =
prepareData(
@@ -157,7 +158,11 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
parameters.buildMetricsService.get().buildOperationRecords,
parameters.additionalTags.get()
)
data?.also { parameters.reportingSettings.get().httpReportSettings?.also { executorService.submit { sendDataViaHttp(it, data, log) } } }
data?.also {
executorService.submit {
httpService.sendData(data, log)
}
}
}
}
@@ -316,6 +321,7 @@ abstract class BuildReportsService : BuildService<BuildReportsService.Parameters
it.parameters.kotlinVersion.set(kotlinVersion)
it.parameters.startParameters.set(getStartParameters(project))
it.parameters.reportingSettings.set(reportingSettings)
reportingSettings.httpReportSettings?.let { httpSettings -> it.parameters.httpService.set(HttpReportServiceImpl(httpSettings)) }
it.parameters.buildMetricsService = buildMetricsService
it.parameters.projectDir.set(project.rootProject.layout.projectDirectory)
@@ -0,0 +1,83 @@
/*
* Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.gradle.report
import com.google.gson.Gson
import org.gradle.api.logging.Logger
import java.io.IOException
import java.io.Serializable
import java.net.HttpURLConnection
import java.net.URL
import java.util.*
import kotlin.system.measureTimeMillis
interface HttpReportService {
fun sendData(data: Any, log: Logger)
}
class HttpReportServiceImpl(
private val url: String,
private val password: String?,
private val user: String?,
) : HttpReportService, Serializable {
constructor(httpSettings: HttpReportSettings) : this(httpSettings.url, httpSettings.password, httpSettings.user)
companion object {
private var invalidUrl = false
private var requestPreviousFailed = false
private fun checkResponseAndLog(connection: HttpURLConnection, log: Logger) {
val isResponseBad = connection.responseCode !in 200..299
if (isResponseBad) {
val message = "Failed to send statistic to ${connection.url} with ${connection.responseCode}: ${connection.responseMessage}"
if (!requestPreviousFailed) {
log.warn(message)
} else {
log.debug(message)
}
requestPreviousFailed = true
}
}
}
override fun sendData(data: Any, log: Logger) {
val elapsedTime = measureTimeMillis {
if (invalidUrl) {
return
}
val connection = try {
URL(url).openConnection() as HttpURLConnection
} catch (e: IOException) {
log.warn("Unable to open connection to ${url}: ${e.message}")
invalidUrl = true
return
}
try {
if (user != null && password != null) {
val auth = Base64.getEncoder()
.encode("${user}:${password}".toByteArray())
.toString(Charsets.UTF_8)
connection.addRequestProperty("Authorization", "Basic $auth")
}
connection.addRequestProperty("Content-Type", "application/json")
connection.requestMethod = "POST"
connection.doOutput = true
connection.outputStream.use {
it.write(Gson().toJson(data).toByteArray())
}
connection.connect()
checkResponseAndLog(connection, log)
} catch (e: Exception) {
log.debug("Unexpected exception happened ${e.message}: ${e.stackTrace}")
checkResponseAndLog(connection, log)
} finally {
connection.disconnect()
}
}
log.debug("Report statistic by http takes $elapsedTime ms")
}
}